google-cloud-bigquery 1.24.0 → 1.29.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +52 -0
- data/CONTRIBUTING.md +2 -2
- data/LOGGING.md +1 -1
- data/lib/google/cloud/bigquery/convert.rb +0 -4
- data/lib/google/cloud/bigquery/copy_job.rb +1 -0
- data/lib/google/cloud/bigquery/data.rb +2 -2
- data/lib/google/cloud/bigquery/dataset.rb +106 -21
- data/lib/google/cloud/bigquery/dataset/access.rb +112 -14
- data/lib/google/cloud/bigquery/dataset/list.rb +2 -2
- data/lib/google/cloud/bigquery/external.rb +328 -3
- data/lib/google/cloud/bigquery/extract_job.rb +8 -10
- data/lib/google/cloud/bigquery/job.rb +43 -3
- data/lib/google/cloud/bigquery/job/list.rb +4 -4
- data/lib/google/cloud/bigquery/load_job.rb +177 -24
- data/lib/google/cloud/bigquery/model/list.rb +2 -2
- data/lib/google/cloud/bigquery/policy.rb +432 -0
- data/lib/google/cloud/bigquery/project.rb +3 -3
- data/lib/google/cloud/bigquery/project/list.rb +2 -2
- data/lib/google/cloud/bigquery/query_job.rb +25 -14
- data/lib/google/cloud/bigquery/routine.rb +128 -9
- data/lib/google/cloud/bigquery/routine/list.rb +2 -2
- data/lib/google/cloud/bigquery/service.rb +44 -13
- data/lib/google/cloud/bigquery/standard_sql.rb +4 -3
- data/lib/google/cloud/bigquery/table.rb +261 -45
- data/lib/google/cloud/bigquery/table/async_inserter.rb +24 -15
- data/lib/google/cloud/bigquery/table/list.rb +2 -2
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +16 -15
@@ -56,7 +56,8 @@ module Google
|
|
56
56
|
# @private The Service object.
|
57
57
|
attr_accessor :service
|
58
58
|
|
59
|
-
attr_reader :name
|
59
|
+
attr_reader :name
|
60
|
+
attr_reader :numeric_id
|
60
61
|
|
61
62
|
##
|
62
63
|
# Creates a new Service instance.
|
@@ -981,8 +982,7 @@ module Google
|
|
981
982
|
# @param [String] description A user-friendly description of the
|
982
983
|
# dataset.
|
983
984
|
# @param [Integer] expiration The default lifetime of all tables in the
|
984
|
-
# dataset, in milliseconds. The minimum value is
|
985
|
-
# (one hour).
|
985
|
+
# dataset, in milliseconds. The minimum value is `3_600_000` (one hour).
|
986
986
|
# @param [String] location The geographic location where the dataset
|
987
987
|
# should reside. Possible values include `EU` and `US`. The default
|
988
988
|
# value is `US`.
|
@@ -121,12 +121,12 @@ module Google
|
|
121
121
|
# puts project.name
|
122
122
|
# end
|
123
123
|
#
|
124
|
-
def all request_limit: nil
|
124
|
+
def all request_limit: nil, &block
|
125
125
|
request_limit = request_limit.to_i if request_limit
|
126
126
|
return enum_for :all, request_limit: request_limit unless block_given?
|
127
127
|
results = self
|
128
128
|
loop do
|
129
|
-
results.each
|
129
|
+
results.each(&block)
|
130
130
|
if request_limit
|
131
131
|
request_limit -= 1
|
132
132
|
break if request_limit.negative?
|
@@ -94,8 +94,7 @@ module Google
|
|
94
94
|
# otherwise.
|
95
95
|
#
|
96
96
|
def batch?
|
97
|
-
|
98
|
-
val == "BATCH"
|
97
|
+
@gapi.configuration.query.priority == "BATCH"
|
99
98
|
end
|
100
99
|
|
101
100
|
##
|
@@ -693,8 +692,11 @@ module Google
|
|
693
692
|
end
|
694
693
|
ensure_schema!
|
695
694
|
|
696
|
-
|
697
|
-
|
695
|
+
data_hash = service.list_tabledata destination_table_dataset_id,
|
696
|
+
destination_table_table_id,
|
697
|
+
token: token,
|
698
|
+
max: max,
|
699
|
+
start: start
|
698
700
|
Data.from_gapi_json data_hash, destination_table_gapi, @gapi, service
|
699
701
|
end
|
700
702
|
alias query_results data
|
@@ -705,12 +707,11 @@ module Google
|
|
705
707
|
##
|
706
708
|
# @private Create an Updater object.
|
707
709
|
def initialize service, gapi
|
710
|
+
super()
|
708
711
|
@service = service
|
709
712
|
@gapi = gapi
|
710
713
|
end
|
711
714
|
|
712
|
-
# rubocop:disable all
|
713
|
-
|
714
715
|
##
|
715
716
|
# @private Create an Updater from an options hash.
|
716
717
|
#
|
@@ -749,8 +750,6 @@ module Google
|
|
749
750
|
updater
|
750
751
|
end
|
751
752
|
|
752
|
-
# rubocop:enable all
|
753
|
-
|
754
753
|
##
|
755
754
|
# Sets the geographic location where the job should run. Required
|
756
755
|
# except for US and EU.
|
@@ -936,13 +935,13 @@ module Google
|
|
936
935
|
raise ArgumentError, "types must use the same format as params" if types.class != params.class
|
937
936
|
|
938
937
|
case params
|
939
|
-
when Array
|
938
|
+
when Array
|
940
939
|
@gapi.configuration.query.use_legacy_sql = false
|
941
940
|
@gapi.configuration.query.parameter_mode = "POSITIONAL"
|
942
941
|
@gapi.configuration.query.query_parameters = params.zip(types).map do |param, type|
|
943
942
|
Convert.to_query_param param, type
|
944
943
|
end
|
945
|
-
when Hash
|
944
|
+
when Hash
|
946
945
|
@gapi.configuration.query.use_legacy_sql = false
|
947
946
|
@gapi.configuration.query.parameter_mode = "NAMED"
|
948
947
|
@gapi.configuration.query.query_parameters = params.map do |name, param|
|
@@ -1593,9 +1592,20 @@ module Google
|
|
1593
1592
|
# end
|
1594
1593
|
#
|
1595
1594
|
class Stage
|
1596
|
-
attr_reader :compute_ratio_avg
|
1597
|
-
|
1598
|
-
|
1595
|
+
attr_reader :compute_ratio_avg
|
1596
|
+
attr_reader :compute_ratio_max
|
1597
|
+
attr_reader :id
|
1598
|
+
attr_reader :name
|
1599
|
+
attr_reader :read_ratio_avg
|
1600
|
+
attr_reader :read_ratio_max
|
1601
|
+
attr_reader :records_read
|
1602
|
+
attr_reader :records_written
|
1603
|
+
attr_reader :status
|
1604
|
+
attr_reader :steps
|
1605
|
+
attr_reader :wait_ratio_avg
|
1606
|
+
attr_reader :wait_ratio_max
|
1607
|
+
attr_reader :write_ratio_avg
|
1608
|
+
attr_reader :write_ratio_max
|
1599
1609
|
|
1600
1610
|
##
|
1601
1611
|
# @private Creates a new Stage instance.
|
@@ -1658,7 +1668,8 @@ module Google
|
|
1658
1668
|
# end
|
1659
1669
|
#
|
1660
1670
|
class Step
|
1661
|
-
attr_reader :kind
|
1671
|
+
attr_reader :kind
|
1672
|
+
attr_reader :substeps
|
1662
1673
|
|
1663
1674
|
##
|
1664
1675
|
# @private Creates a new Stage instance.
|
@@ -603,6 +603,93 @@ module Google
|
|
603
603
|
update_gapi!
|
604
604
|
end
|
605
605
|
|
606
|
+
###
|
607
|
+
# The JavaScript UDF determinism level. Optional.
|
608
|
+
#
|
609
|
+
# * `DETERMINISTIC` - Deterministic indicates that two calls with the same input to a UDF yield the same output.
|
610
|
+
# If all JavaScript UDFs are `DETERMINISTIC`, the query result is potentially cachable.
|
611
|
+
# * `NOT_DETERMINISTIC` - Not deterministic indicates that the output of the UDF is not guaranteed to yield the
|
612
|
+
# same output each time for a given set of inputs. If any JavaScript UDF is `NOT_DETERMINISTIC`, the query
|
613
|
+
# result is not cacheable.
|
614
|
+
#
|
615
|
+
# Even if a JavaScript UDF is deterministic, many other factors can prevent usage of cached query results.
|
616
|
+
# Example factors include but not limited to: DDL/DML, non-deterministic SQL function calls, update of
|
617
|
+
# referenced tables/views/UDFs or imported JavaScript libraries. SQL UDFs cannot have determinism specified.
|
618
|
+
# Their determinism is automatically determined.
|
619
|
+
#
|
620
|
+
# @return [String, nil] The routine determinism level in upper case, or `nil` if not set or the object is a
|
621
|
+
# reference (see {#reference?}).
|
622
|
+
#
|
623
|
+
# @example
|
624
|
+
# require "google/cloud/bigquery"
|
625
|
+
#
|
626
|
+
# bigquery = Google::Cloud::Bigquery.new
|
627
|
+
# dataset = bigquery.dataset "my_dataset"
|
628
|
+
# routine = dataset.routine "my_routine"
|
629
|
+
#
|
630
|
+
# routine.determinism_level #=> "NOT_DETERMINISTIC"
|
631
|
+
#
|
632
|
+
# @!group Attributes
|
633
|
+
#
|
634
|
+
def determinism_level
|
635
|
+
return nil if reference?
|
636
|
+
ensure_full_data!
|
637
|
+
@gapi.determinism_level
|
638
|
+
end
|
639
|
+
|
640
|
+
##
|
641
|
+
# Updates the JavaScript UDF determinism level. Optional.
|
642
|
+
#
|
643
|
+
# * `DETERMINISTIC` - Deterministic indicates that two calls with the same input to a UDF yield the same output.
|
644
|
+
# If all JavaScript UDFs are `DETERMINISTIC`, the query result is potentially cachable.
|
645
|
+
# * `NOT_DETERMINISTIC` - Not deterministic indicates that the output of the UDF is not guaranteed to yield the
|
646
|
+
# same output each time for a given set of inputs. If any JavaScript UDF is `NOT_DETERMINISTIC`, the query
|
647
|
+
# result is not cacheable.
|
648
|
+
#
|
649
|
+
# @param [String, nil] new_determinism_level The new routine determinism level in upper case.
|
650
|
+
#
|
651
|
+
# @example
|
652
|
+
# require "google/cloud/bigquery"
|
653
|
+
#
|
654
|
+
# bigquery = Google::Cloud::Bigquery.new
|
655
|
+
# dataset = bigquery.dataset "my_dataset"
|
656
|
+
# routine = dataset.routine "my_routine"
|
657
|
+
#
|
658
|
+
# routine.determinism_level #=> "NOT_DETERMINISTIC"
|
659
|
+
# routine.determinism_level = "DETERMINISTIC"
|
660
|
+
#
|
661
|
+
# @!group Attributes
|
662
|
+
#
|
663
|
+
def determinism_level= new_determinism_level
|
664
|
+
ensure_full_data!
|
665
|
+
@gapi.determinism_level = new_determinism_level
|
666
|
+
update_gapi!
|
667
|
+
end
|
668
|
+
|
669
|
+
##
|
670
|
+
# Checks if the value of {#determinism_level} is `DETERMINISTIC`. The default is `false`.
|
671
|
+
#
|
672
|
+
# @return [Boolean] `true` when `DETERMINISTIC` and the object is not a reference (see {#reference?}), `false`
|
673
|
+
# otherwise.
|
674
|
+
#
|
675
|
+
# @!group Attributes
|
676
|
+
#
|
677
|
+
def determinism_level_deterministic?
|
678
|
+
@gapi.determinism_level == "DETERMINISTIC"
|
679
|
+
end
|
680
|
+
|
681
|
+
##
|
682
|
+
# Checks if the value of {#determinism_level} is `NOT_DETERMINISTIC`. The default is `false`.
|
683
|
+
#
|
684
|
+
# @return [Boolean] `true` when `NOT_DETERMINISTIC` and the object is not a reference (see {#reference?}),
|
685
|
+
# `false` otherwise.
|
686
|
+
#
|
687
|
+
# @!group Attributes
|
688
|
+
#
|
689
|
+
def determinism_level_not_deterministic?
|
690
|
+
@gapi.determinism_level == "NOT_DETERMINISTIC"
|
691
|
+
end
|
692
|
+
|
606
693
|
##
|
607
694
|
# Updates the routine with changes made in the given block in a single update request. The following attributes
|
608
695
|
# may be set: {Updater#routine_type=}, {Updater#language=}, {Updater#arguments=}, {Updater#return_type=},
|
@@ -919,6 +1006,7 @@ module Google
|
|
919
1006
|
##
|
920
1007
|
# @private Create an Updater object.
|
921
1008
|
def initialize gapi
|
1009
|
+
super()
|
922
1010
|
@original_gapi = gapi
|
923
1011
|
@gapi = gapi.dup
|
924
1012
|
end
|
@@ -999,7 +1087,9 @@ module Google
|
|
999
1087
|
# routine = dataset.routine "my_routine"
|
1000
1088
|
#
|
1001
1089
|
# routine.return_type.type_kind #=> "INT64"
|
1002
|
-
# routine.
|
1090
|
+
# routine.update do |r|
|
1091
|
+
# r.return_type = "STRING"
|
1092
|
+
# end
|
1003
1093
|
#
|
1004
1094
|
def return_type= new_return_type
|
1005
1095
|
@gapi.return_type = StandardSql::DataType.gapi_from_string_or_data_type new_return_type
|
@@ -1019,9 +1109,11 @@ module Google
|
|
1019
1109
|
# dataset = bigquery.dataset "my_dataset"
|
1020
1110
|
# routine = dataset.routine "my_routine"
|
1021
1111
|
#
|
1022
|
-
# routine.
|
1023
|
-
#
|
1024
|
-
#
|
1112
|
+
# routine.update do |r|
|
1113
|
+
# r.imported_libraries = [
|
1114
|
+
# "gs://cloud-samples-data/bigquery/udfs/max-value.js"
|
1115
|
+
# ]
|
1116
|
+
# end
|
1025
1117
|
#
|
1026
1118
|
def imported_libraries= new_imported_libraries
|
1027
1119
|
@gapi.imported_libraries = new_imported_libraries
|
@@ -1069,12 +1161,43 @@ module Google
|
|
1069
1161
|
# routine = dataset.routine "my_routine"
|
1070
1162
|
#
|
1071
1163
|
# routine.description #=> "My routine description"
|
1072
|
-
# routine.
|
1164
|
+
# routine.update do |r|
|
1165
|
+
# r.description = "My updated routine description"
|
1166
|
+
# end
|
1073
1167
|
#
|
1074
1168
|
def description= new_description
|
1075
1169
|
@gapi.description = new_description
|
1076
1170
|
end
|
1077
1171
|
|
1172
|
+
##
|
1173
|
+
# Updates the JavaScript UDF determinism level. Optional.
|
1174
|
+
#
|
1175
|
+
# * `DETERMINISTIC` - Deterministic indicates that two calls with the same input to a UDF yield the same
|
1176
|
+
# output. If all JavaScript UDFs are `DETERMINISTIC`, the query result is potentially cachable.
|
1177
|
+
# * `NOT_DETERMINISTIC` - Not deterministic indicates that the output of the UDF is not guaranteed to yield
|
1178
|
+
# the same output each time for a given set of inputs. If any JavaScript UDF is `NOT_DETERMINISTIC`, the
|
1179
|
+
# query result is not cacheable.
|
1180
|
+
#
|
1181
|
+
# @param [String, nil] new_determinism_level The new routine determinism level in upper case.
|
1182
|
+
#
|
1183
|
+
# @example
|
1184
|
+
# require "google/cloud/bigquery"
|
1185
|
+
#
|
1186
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1187
|
+
# dataset = bigquery.dataset "my_dataset"
|
1188
|
+
# routine = dataset.routine "my_routine"
|
1189
|
+
#
|
1190
|
+
# routine.determinism_level #=> "NOT_DETERMINISTIC"
|
1191
|
+
# routine.update do |r|
|
1192
|
+
# r.determinism_level = "DETERMINISTIC"
|
1193
|
+
# end
|
1194
|
+
#
|
1195
|
+
# @!group Attributes
|
1196
|
+
#
|
1197
|
+
def determinism_level= new_determinism_level
|
1198
|
+
@gapi.determinism_level = new_determinism_level
|
1199
|
+
end
|
1200
|
+
|
1078
1201
|
def update
|
1079
1202
|
raise "not implemented in #{self.class}"
|
1080
1203
|
end
|
@@ -1088,15 +1211,11 @@ module Google
|
|
1088
1211
|
end
|
1089
1212
|
alias refresh! reload!
|
1090
1213
|
|
1091
|
-
# rubocop:disable Style/CaseEquality
|
1092
|
-
|
1093
1214
|
# @private
|
1094
1215
|
def updates?
|
1095
1216
|
!(@gapi === @original_gapi)
|
1096
1217
|
end
|
1097
1218
|
|
1098
|
-
# rubocop:enable Style/CaseEquality
|
1099
|
-
|
1100
1219
|
# @private
|
1101
1220
|
def to_gapi
|
1102
1221
|
@gapi
|
@@ -124,12 +124,12 @@ module Google
|
|
124
124
|
# puts routine.routine_id
|
125
125
|
# end
|
126
126
|
#
|
127
|
-
def all request_limit: nil
|
127
|
+
def all request_limit: nil, &block
|
128
128
|
request_limit = request_limit.to_i if request_limit
|
129
129
|
return enum_for :all, request_limit: request_limit unless block_given?
|
130
130
|
results = self
|
131
131
|
loop do
|
132
|
-
results.each
|
132
|
+
results.each(&block)
|
133
133
|
if request_limit
|
134
134
|
request_limit -= 1
|
135
135
|
break if request_limit.negative?
|
@@ -182,6 +182,34 @@ module Google
|
|
182
182
|
end
|
183
183
|
end
|
184
184
|
|
185
|
+
##
|
186
|
+
# Returns Google::Apis::BigqueryV2::Policy
|
187
|
+
def get_table_policy dataset_id, table_id
|
188
|
+
policy_options = API::GetPolicyOptions.new requested_policy_version: 1
|
189
|
+
execute do
|
190
|
+
service.get_table_iam_policy table_path(dataset_id, table_id),
|
191
|
+
API::GetIamPolicyRequest.new(options: policy_options)
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
##
|
196
|
+
# @param [Google::Apis::BigqueryV2::Policy] new_policy
|
197
|
+
def set_table_policy dataset_id, table_id, new_policy
|
198
|
+
execute do
|
199
|
+
service.set_table_iam_policy table_path(dataset_id, table_id),
|
200
|
+
API::SetIamPolicyRequest.new(policy: new_policy)
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
##
|
205
|
+
# Returns Google::Apis::BigqueryV2::TestIamPermissionsResponse
|
206
|
+
def test_table_permissions dataset_id, table_id, permissions
|
207
|
+
execute do
|
208
|
+
service.test_table_iam_permissions table_path(dataset_id, table_id),
|
209
|
+
API::TestIamPermissionsRequest.new(permissions: permissions)
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
185
213
|
##
|
186
214
|
# Deletes the table specified by tableId from the dataset.
|
187
215
|
# If the table contains data, all the data will be deleted.
|
@@ -455,7 +483,7 @@ module Google
|
|
455
483
|
table_id: m["tbl"]
|
456
484
|
}.delete_if { |_, v| v.nil? }
|
457
485
|
str_table_ref_hash = default_ref.to_h.merge str_table_ref_hash
|
458
|
-
ref = Google::Apis::BigqueryV2::TableReference.new
|
486
|
+
ref = Google::Apis::BigqueryV2::TableReference.new(**str_table_ref_hash)
|
459
487
|
validate_table_ref ref
|
460
488
|
ref
|
461
489
|
end
|
@@ -508,6 +536,11 @@ module Google
|
|
508
536
|
|
509
537
|
protected
|
510
538
|
|
539
|
+
# Creates a formatted table path.
|
540
|
+
def table_path dataset_id, table_id
|
541
|
+
"projects/#{@project}/datasets/#{dataset_id}/tables/#{table_id}"
|
542
|
+
end
|
543
|
+
|
511
544
|
# Generate a random string similar to the BigQuery service job IDs.
|
512
545
|
def generate_id
|
513
546
|
SecureRandom.urlsafe_base64 21
|
@@ -521,9 +554,9 @@ module Google
|
|
521
554
|
nil
|
522
555
|
end
|
523
556
|
|
524
|
-
def execute backoff: nil
|
557
|
+
def execute backoff: nil, &block
|
525
558
|
if backoff
|
526
|
-
Backoff.new(retries: retries).execute
|
559
|
+
Backoff.new(retries: retries).execute(&block)
|
527
560
|
else
|
528
561
|
yield
|
529
562
|
end
|
@@ -557,22 +590,20 @@ module Google
|
|
557
590
|
def execute
|
558
591
|
current_retries = 0
|
559
592
|
loop do
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
564
|
-
|
565
|
-
|
566
|
-
current_retries += 1
|
567
|
-
end
|
593
|
+
return yield
|
594
|
+
rescue Google::Apis::Error => e
|
595
|
+
raise e unless retry? e.body, current_retries
|
596
|
+
|
597
|
+
@backoff.call current_retries
|
598
|
+
current_retries += 1
|
568
599
|
end
|
569
600
|
end
|
570
601
|
|
571
602
|
protected
|
572
603
|
|
573
604
|
def retry? result, current_retries #:nodoc:
|
574
|
-
if current_retries < @retries
|
575
|
-
return true
|
605
|
+
if current_retries < @retries && retry_error_reason?(result)
|
606
|
+
return true
|
576
607
|
end
|
577
608
|
false
|
578
609
|
end
|
@@ -401,11 +401,12 @@ module Google
|
|
401
401
|
# @private New Google::Apis::BigqueryV2::StandardSqlDataType from a String or StandardSql::DataType object.
|
402
402
|
def self.gapi_from_string_or_data_type data_type
|
403
403
|
return if data_type.nil?
|
404
|
-
|
404
|
+
case data_type
|
405
|
+
when StandardSql::DataType
|
405
406
|
data_type.to_gapi
|
406
|
-
|
407
|
+
when Hash
|
407
408
|
data_type
|
408
|
-
|
409
|
+
when String, Symbol
|
409
410
|
Google::Apis::BigqueryV2::StandardSqlDataType.new type_kind: data_type.to_s.upcase
|
410
411
|
else
|
411
412
|
raise ArgumentError, "Unable to convert #{data_type} to Google::Apis::BigqueryV2::StandardSqlDataType"
|
@@ -23,6 +23,7 @@ require "google/cloud/bigquery/external"
|
|
23
23
|
require "google/cloud/bigquery/insert_response"
|
24
24
|
require "google/cloud/bigquery/table/async_inserter"
|
25
25
|
require "google/cloud/bigquery/convert"
|
26
|
+
require "google/cloud/bigquery/policy"
|
26
27
|
require "google/apis/bigquery_v2"
|
27
28
|
|
28
29
|
module Google
|
@@ -36,16 +37,16 @@ module Google
|
|
36
37
|
# repeated fields.
|
37
38
|
#
|
38
39
|
# The Table class can also represent a
|
39
|
-
# [view](https://cloud.google.com/bigquery/docs/views), which is a virtual
|
40
|
-
# table defined by a SQL query
|
41
|
-
# materialized views,
|
42
|
-
#
|
43
|
-
#
|
44
|
-
# indirectly by the top-level query. (See {#view?}, {#query}, {#query=},
|
45
|
-
# and {Dataset#create_view}.)
|
40
|
+
# [logical view](https://cloud.google.com/bigquery/docs/views), which is a virtual
|
41
|
+
# table defined by a SQL query (see {#view?} and {Dataset#create_view}); or a
|
42
|
+
# [materialized view](https://cloud.google.com/bigquery/docs/materialized-views-intro),
|
43
|
+
# which is a precomputed view that periodically caches results of a query for increased
|
44
|
+
# performance and efficiency (see {#materialized_view?} and {Dataset#create_materialized_view}).
|
46
45
|
#
|
47
46
|
# @see https://cloud.google.com/bigquery/docs/loading-data#loading_denormalized_nested_and_repeated_data
|
48
47
|
# Loading denormalized, nested, and repeated data
|
48
|
+
# @see https://cloud.google.com/bigquery/docs/views Creating views
|
49
|
+
# @see https://cloud.google.com/bigquery/docs/materialized-views-intro Introduction to materialized views
|
49
50
|
#
|
50
51
|
# @example
|
51
52
|
# require "google/cloud/bigquery"
|
@@ -76,7 +77,7 @@ module Google
|
|
76
77
|
# }
|
77
78
|
# table.insert row
|
78
79
|
#
|
79
|
-
# @example Creating a
|
80
|
+
# @example Creating a logical view:
|
80
81
|
# require "google/cloud/bigquery"
|
81
82
|
#
|
82
83
|
# bigquery = Google::Cloud::Bigquery.new
|
@@ -85,6 +86,15 @@ module Google
|
|
85
86
|
# "SELECT name, age FROM `my_project.my_dataset.my_table`"
|
86
87
|
# view.view? # true
|
87
88
|
#
|
89
|
+
# @example Creating a materialized view:
|
90
|
+
# require "google/cloud/bigquery"
|
91
|
+
#
|
92
|
+
# bigquery = Google::Cloud::Bigquery.new
|
93
|
+
# dataset = bigquery.dataset "my_dataset"
|
94
|
+
# view = dataset.create_materialized_view "my_materialized_view",
|
95
|
+
# "SELECT name, age FROM `my_project.my_dataset.my_table`"
|
96
|
+
# view.materialized_view? # true
|
97
|
+
#
|
88
98
|
class Table
|
89
99
|
##
|
90
100
|
# @private The Service object.
|
@@ -725,7 +735,7 @@ module Google
|
|
725
735
|
end
|
726
736
|
|
727
737
|
##
|
728
|
-
# Checks if the table's type is
|
738
|
+
# Checks if the table's type is `TABLE`.
|
729
739
|
#
|
730
740
|
# @return [Boolean, nil] `true` when the type is `TABLE`, `false`
|
731
741
|
# otherwise, if the object is a resource (see {#resource?}); `nil` if
|
@@ -739,8 +749,10 @@ module Google
|
|
739
749
|
end
|
740
750
|
|
741
751
|
##
|
742
|
-
# Checks if the table's type is
|
743
|
-
# represents a BigQuery view. See {Dataset#create_view}.
|
752
|
+
# Checks if the table's type is `VIEW`, indicating that the table
|
753
|
+
# represents a BigQuery logical view. See {Dataset#create_view}.
|
754
|
+
#
|
755
|
+
# @see https://cloud.google.com/bigquery/docs/views Creating views
|
744
756
|
#
|
745
757
|
# @return [Boolean, nil] `true` when the type is `VIEW`, `false`
|
746
758
|
# otherwise, if the object is a resource (see {#resource?}); `nil` if
|
@@ -754,7 +766,25 @@ module Google
|
|
754
766
|
end
|
755
767
|
|
756
768
|
##
|
757
|
-
# Checks if the table's type is
|
769
|
+
# Checks if the table's type is `MATERIALIZED_VIEW`, indicating that
|
770
|
+
# the table represents a BigQuery materialized view.
|
771
|
+
# See {Dataset#create_materialized_view}.
|
772
|
+
#
|
773
|
+
# @see https://cloud.google.com/bigquery/docs/materialized-views-intro Introduction to materialized views
|
774
|
+
#
|
775
|
+
# @return [Boolean, nil] `true` when the type is `MATERIALIZED_VIEW`,
|
776
|
+
# `false` otherwise, if the object is a resource (see {#resource?});
|
777
|
+
# `nil` if the object is a reference (see {#reference?}).
|
778
|
+
#
|
779
|
+
# @!group Attributes
|
780
|
+
#
|
781
|
+
def materialized_view?
|
782
|
+
return nil if reference?
|
783
|
+
@gapi.type == "MATERIALIZED_VIEW"
|
784
|
+
end
|
785
|
+
|
786
|
+
##
|
787
|
+
# Checks if the table's type is `EXTERNAL`, indicating that the table
|
758
788
|
# represents an External Data Source. See {#external?} and
|
759
789
|
# {External::DataSource}.
|
760
790
|
#
|
@@ -1137,21 +1167,24 @@ module Google
|
|
1137
1167
|
end
|
1138
1168
|
|
1139
1169
|
##
|
1140
|
-
# The query that
|
1170
|
+
# The query that defines the view or materialized view. See {#view?} and
|
1171
|
+
# {#materialized_view?}.
|
1141
1172
|
#
|
1142
|
-
# @return [String] The query that defines the view
|
1173
|
+
# @return [String, nil] The query that defines the view or materialized_view;
|
1174
|
+
# or `nil` if not a view or materialized view.
|
1143
1175
|
#
|
1144
1176
|
# @!group Attributes
|
1145
1177
|
#
|
1146
1178
|
def query
|
1147
|
-
@gapi.view&.query
|
1179
|
+
view? ? @gapi.view&.query : @gapi.materialized_view&.query
|
1148
1180
|
end
|
1149
1181
|
|
1150
1182
|
##
|
1151
|
-
# Updates the query that
|
1183
|
+
# Updates the query that defines the view. (See {#view?}.) Not supported
|
1184
|
+
# for materialized views.
|
1152
1185
|
#
|
1153
|
-
# This sets the query using standard SQL. To specify legacy SQL or
|
1154
|
-
# use user-defined function resources use (#set_query) instead.
|
1186
|
+
# This method sets the query using standard SQL. To specify legacy SQL or
|
1187
|
+
# to use user-defined function resources for a view, use (#set_query) instead.
|
1155
1188
|
#
|
1156
1189
|
# @see https://cloud.google.com/bigquery/query-reference BigQuery Query
|
1157
1190
|
# Reference
|
@@ -1166,7 +1199,7 @@ module Google
|
|
1166
1199
|
# view = dataset.table "my_view"
|
1167
1200
|
#
|
1168
1201
|
# view.query = "SELECT first_name FROM " \
|
1169
|
-
#
|
1202
|
+
# "`my_project.my_dataset.my_table`"
|
1170
1203
|
#
|
1171
1204
|
# @!group Lifecycle
|
1172
1205
|
#
|
@@ -1175,12 +1208,12 @@ module Google
|
|
1175
1208
|
end
|
1176
1209
|
|
1177
1210
|
##
|
1178
|
-
# Updates the query that
|
1179
|
-
#
|
1180
|
-
# resources.
|
1211
|
+
# Updates the query that defines the view. (See {#view?}.) Not supported for
|
1212
|
+
# materialized views.
|
1181
1213
|
#
|
1182
|
-
#
|
1183
|
-
#
|
1214
|
+
# Allows setting of standard vs. legacy SQL and user-defined function resources.
|
1215
|
+
#
|
1216
|
+
# @see https://cloud.google.com/bigquery/query-reference BigQuery Query Reference
|
1184
1217
|
#
|
1185
1218
|
# @param [String] query The query that defines the view.
|
1186
1219
|
# @param [Boolean] standard_sql Specifies whether to use BigQuery's
|
@@ -1192,11 +1225,12 @@ module Google
|
|
1192
1225
|
# SQL](https://cloud.google.com/bigquery/docs/reference/legacy-sql)
|
1193
1226
|
# dialect. Optional. The default value is false.
|
1194
1227
|
# @param [Array<String>, String] udfs User-defined function resources
|
1195
|
-
# used in a legacy SQL query.
|
1196
|
-
#
|
1197
|
-
#
|
1198
|
-
#
|
1199
|
-
#
|
1228
|
+
# used in a legacy SQL query. Optional.
|
1229
|
+
#
|
1230
|
+
# May be either a code resource to load from a Google Cloud Storage URI
|
1231
|
+
# (`gs://bucket/path`), or an inline resource that contains code for a
|
1232
|
+
# user-defined function (UDF). Providing an inline code resource is equivalent
|
1233
|
+
# to providing a URI for a file containing the same code.
|
1200
1234
|
#
|
1201
1235
|
# This parameter is used for defining User Defined Function (UDF)
|
1202
1236
|
# resources only when using legacy SQL. Users of standard SQL should
|
@@ -1207,7 +1241,7 @@ module Google
|
|
1207
1241
|
# standard SQL - Differences in user-defined JavaScript
|
1208
1242
|
# functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/migrating-from-legacy-sql#differences_in_user-defined_javascript_functions)
|
1209
1243
|
#
|
1210
|
-
# @example
|
1244
|
+
# @example Update a view:
|
1211
1245
|
# require "google/cloud/bigquery"
|
1212
1246
|
#
|
1213
1247
|
# bigquery = Google::Cloud::Bigquery.new
|
@@ -1215,12 +1249,13 @@ module Google
|
|
1215
1249
|
# view = dataset.table "my_view"
|
1216
1250
|
#
|
1217
1251
|
# view.set_query "SELECT first_name FROM " \
|
1218
|
-
#
|
1252
|
+
# "`my_project.my_dataset.my_table`",
|
1219
1253
|
# standard_sql: true
|
1220
1254
|
#
|
1221
1255
|
# @!group Lifecycle
|
1222
1256
|
#
|
1223
1257
|
def set_query query, standard_sql: nil, legacy_sql: nil, udfs: nil
|
1258
|
+
raise "Updating the query is not supported for Table type: #{@gapi.type}" unless view?
|
1224
1259
|
use_legacy_sql = Convert.resolve_legacy_sql standard_sql, legacy_sql
|
1225
1260
|
@gapi.view = Google::Apis::BigqueryV2::ViewDefinition.new(
|
1226
1261
|
query: query,
|
@@ -1231,26 +1266,28 @@ module Google
|
|
1231
1266
|
end
|
1232
1267
|
|
1233
1268
|
##
|
1234
|
-
# Checks if the view's query is using legacy sql.
|
1269
|
+
# Checks if the view's query is using legacy sql. See {#view?}.
|
1235
1270
|
#
|
1236
|
-
# @return [Boolean] `true` when legacy sql is used, `false` otherwise.
|
1271
|
+
# @return [Boolean] `true` when legacy sql is used, `false` otherwise; or `nil` if not a logical view.
|
1237
1272
|
#
|
1238
1273
|
# @!group Attributes
|
1239
1274
|
#
|
1240
1275
|
def query_legacy_sql?
|
1276
|
+
return nil unless @gapi.view
|
1241
1277
|
val = @gapi.view.use_legacy_sql
|
1242
1278
|
return true if val.nil?
|
1243
1279
|
val
|
1244
1280
|
end
|
1245
1281
|
|
1246
1282
|
##
|
1247
|
-
# Checks if the view's query is using standard sql.
|
1283
|
+
# Checks if the view's query is using standard sql. See {#view?}.
|
1248
1284
|
#
|
1249
1285
|
# @return [Boolean] `true` when standard sql is used, `false` otherwise.
|
1250
1286
|
#
|
1251
1287
|
# @!group Attributes
|
1252
1288
|
#
|
1253
1289
|
def query_standard_sql?
|
1290
|
+
return nil unless @gapi.view
|
1254
1291
|
!query_legacy_sql?
|
1255
1292
|
end
|
1256
1293
|
|
@@ -1262,18 +1299,193 @@ module Google
|
|
1262
1299
|
# equivalent to providing a URI for a file containing the same code. See
|
1263
1300
|
# [User-Defined
|
1264
1301
|
# Functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/user-defined-functions).
|
1302
|
+
# See {#view?}.
|
1265
1303
|
#
|
1266
|
-
# @return [Array<String
|
1267
|
-
# and/or inline source code.
|
1304
|
+
# @return [Array<String>, nil] An array containing Google Cloud Storage URIs
|
1305
|
+
# and/or inline source code, or `nil` if not a logical view.
|
1268
1306
|
#
|
1269
1307
|
# @!group Attributes
|
1270
1308
|
#
|
1271
1309
|
def query_udfs
|
1310
|
+
return nil unless @gapi.view
|
1272
1311
|
udfs_gapi = @gapi.view.user_defined_function_resources
|
1273
1312
|
return [] if udfs_gapi.nil?
|
1274
1313
|
Array(udfs_gapi).map { |udf| udf.inline_code || udf.resource_uri }
|
1275
1314
|
end
|
1276
1315
|
|
1316
|
+
##
|
1317
|
+
# Whether automatic refresh of the materialized view is enabled. When true,
|
1318
|
+
# the materialized view is updated when the base table is updated. The default
|
1319
|
+
# value is true. See {#materialized_view?}.
|
1320
|
+
#
|
1321
|
+
# @return [Boolean, nil] `true` when automatic refresh is enabled, `false` otherwise;
|
1322
|
+
# or `nil` if not a materialized view.
|
1323
|
+
#
|
1324
|
+
# @!group Attributes
|
1325
|
+
#
|
1326
|
+
def enable_refresh?
|
1327
|
+
return nil unless @gapi.materialized_view
|
1328
|
+
val = @gapi.materialized_view.enable_refresh
|
1329
|
+
return true if val.nil?
|
1330
|
+
val
|
1331
|
+
end
|
1332
|
+
|
1333
|
+
##
|
1334
|
+
# Sets whether automatic refresh of the materialized view is enabled. When true,
|
1335
|
+
# the materialized view is updated when the base table is updated. See {#materialized_view?}.
|
1336
|
+
#
|
1337
|
+
# @param [Boolean] new_enable_refresh `true` when automatic refresh is enabled, `false` otherwise.
|
1338
|
+
#
|
1339
|
+
# @!group Attributes
|
1340
|
+
#
|
1341
|
+
def enable_refresh= new_enable_refresh
|
1342
|
+
@gapi.materialized_view = Google::Apis::BigqueryV2::MaterializedViewDefinition.new(
|
1343
|
+
enable_refresh: new_enable_refresh
|
1344
|
+
)
|
1345
|
+
patch_gapi! :materialized_view
|
1346
|
+
end
|
1347
|
+
|
1348
|
+
##
|
1349
|
+
# The time when the materialized view was last modified.
|
1350
|
+
# See {#materialized_view?}.
|
1351
|
+
#
|
1352
|
+
# @return [Time, nil] The time, or `nil` if not present or not a materialized view.
|
1353
|
+
#
|
1354
|
+
# @!group Attributes
|
1355
|
+
#
|
1356
|
+
def last_refresh_time
|
1357
|
+
Convert.millis_to_time @gapi.materialized_view&.last_refresh_time
|
1358
|
+
end
|
1359
|
+
|
1360
|
+
##
|
1361
|
+
# The maximum frequency in milliseconds at which the materialized view will be refreshed.
|
1362
|
+
# See {#materialized_view?}.
|
1363
|
+
#
|
1364
|
+
# @return [Integer, nil] The maximum frequency in milliseconds;
|
1365
|
+
# or `nil` if not a materialized view.
|
1366
|
+
#
|
1367
|
+
# @!group Attributes
|
1368
|
+
#
|
1369
|
+
def refresh_interval_ms
|
1370
|
+
@gapi.materialized_view&.refresh_interval_ms
|
1371
|
+
end
|
1372
|
+
|
1373
|
+
##
|
1374
|
+
# Sets the maximum frequency at which the materialized view will be refreshed.
|
1375
|
+
# See {#materialized_view?}.
|
1376
|
+
#
|
1377
|
+
# @param [Integer] new_refresh_interval_ms The maximum frequency in milliseconds.
|
1378
|
+
#
|
1379
|
+
# @!group Attributes
|
1380
|
+
#
|
1381
|
+
def refresh_interval_ms= new_refresh_interval_ms
|
1382
|
+
@gapi.materialized_view = Google::Apis::BigqueryV2::MaterializedViewDefinition.new(
|
1383
|
+
refresh_interval_ms: new_refresh_interval_ms
|
1384
|
+
)
|
1385
|
+
patch_gapi! :materialized_view
|
1386
|
+
end
|
1387
|
+
|
1388
|
+
##
|
1389
|
+
# Gets the Cloud IAM access control policy for the table. The latest policy will be read from the service. See
|
1390
|
+
# also {#update_policy}.
|
1391
|
+
#
|
1392
|
+
# @see https://cloud.google.com/iam/docs/managing-policies Managing Policies
|
1393
|
+
# @see https://cloud.google.com/bigquery/docs/table-access-controls-intro Controlling access to tables
|
1394
|
+
#
|
1395
|
+
# @return [Policy] The frozen policy for the table.
|
1396
|
+
#
|
1397
|
+
# @example
|
1398
|
+
# require "google/cloud/bigquery"
|
1399
|
+
#
|
1400
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1401
|
+
# dataset = bigquery.dataset "my_dataset"
|
1402
|
+
# table = dataset.table "my_table"
|
1403
|
+
#
|
1404
|
+
# policy = table.policy
|
1405
|
+
#
|
1406
|
+
# policy.frozen? #=> true
|
1407
|
+
# binding_owner = policy.bindings.find { |b| b.role == "roles/owner" }
|
1408
|
+
# binding_owner.role #=> "roles/owner"
|
1409
|
+
# binding_owner.members #=> ["user:owner@example.com"]
|
1410
|
+
# binding_owner.frozen? #=> true
|
1411
|
+
# binding_owner.members.frozen? #=> true
|
1412
|
+
#
|
1413
|
+
def policy
|
1414
|
+
raise ArgumentError, "Block argument not supported: Use #update_policy instead." if block_given?
|
1415
|
+
ensure_service!
|
1416
|
+
gapi = service.get_table_policy dataset_id, table_id
|
1417
|
+
Policy.from_gapi(gapi).freeze
|
1418
|
+
end
|
1419
|
+
|
1420
|
+
##
|
1421
|
+
# Updates the Cloud IAM access control policy for the table. The latest policy will be read from the service.
|
1422
|
+
# See also {#policy}.
|
1423
|
+
#
|
1424
|
+
# @see https://cloud.google.com/iam/docs/managing-policies Managing Policies
|
1425
|
+
# @see https://cloud.google.com/bigquery/docs/table-access-controls-intro Controlling access to tables
|
1426
|
+
#
|
1427
|
+
# @yield [policy] A block for updating the policy. The latest policy will be read from the service and passed to
|
1428
|
+
# the block. After the block completes, the modified policy will be written to the service.
|
1429
|
+
# @yieldparam [Policy] policy The mutable Policy for the table.
|
1430
|
+
#
|
1431
|
+
# @return [Policy] The updated and frozen policy for the table.
|
1432
|
+
#
|
1433
|
+
# @example Update the policy by passing a block.
|
1434
|
+
# require "google/cloud/bigquery"
|
1435
|
+
#
|
1436
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1437
|
+
# dataset = bigquery.dataset "my_dataset"
|
1438
|
+
# table = dataset.table "my_table"
|
1439
|
+
#
|
1440
|
+
# table.update_policy do |p|
|
1441
|
+
# p.grant role: "roles/viewer", members: "user:viewer@example.com"
|
1442
|
+
# p.revoke role: "roles/editor", members: "user:editor@example.com"
|
1443
|
+
# p.revoke role: "roles/owner"
|
1444
|
+
# end # 2 API calls
|
1445
|
+
#
|
1446
|
+
def update_policy
|
1447
|
+
raise ArgumentError, "A block updating the policy must be provided" unless block_given?
|
1448
|
+
ensure_service!
|
1449
|
+
gapi = service.get_table_policy dataset_id, table_id
|
1450
|
+
policy = Policy.from_gapi gapi
|
1451
|
+
yield policy
|
1452
|
+
# TODO: Check for changes before calling RPC
|
1453
|
+
gapi = service.set_table_policy dataset_id, table_id, policy.to_gapi
|
1454
|
+
Policy.from_gapi(gapi).freeze
|
1455
|
+
end
|
1456
|
+
|
1457
|
+
##
|
1458
|
+
# Tests the specified permissions against the [Cloud
|
1459
|
+
# IAM](https://cloud.google.com/iam/) access control policy.
|
1460
|
+
#
|
1461
|
+
# @see https://cloud.google.com/iam/docs/managing-policies Managing Policies
|
1462
|
+
#
|
1463
|
+
# @param [String, Array<String>] permissions The set of permissions
|
1464
|
+
# against which to check access. Permissions must be of the format
|
1465
|
+
# `bigquery.resource.capability`.
|
1466
|
+
# See https://cloud.google.com/bigquery/docs/access-control#bigquery.
|
1467
|
+
#
|
1468
|
+
# @return [Array<String>] The frozen array of permissions held by the caller.
|
1469
|
+
#
|
1470
|
+
# @example
|
1471
|
+
# require "google/cloud/bigquery"
|
1472
|
+
#
|
1473
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1474
|
+
# dataset = bigquery.dataset "my_dataset"
|
1475
|
+
# table = dataset.table "my_table"
|
1476
|
+
#
|
1477
|
+
# permissions = table.test_iam_permissions "bigquery.tables.get",
|
1478
|
+
# "bigquery.tables.delete"
|
1479
|
+
# permissions.include? "bigquery.tables.get" #=> true
|
1480
|
+
# permissions.include? "bigquery.tables.delete" #=> false
|
1481
|
+
#
|
1482
|
+
def test_iam_permissions *permissions
|
1483
|
+
permissions = Array(permissions).flatten
|
1484
|
+
ensure_service!
|
1485
|
+
gapi = service.test_table_permissions dataset_id, table_id, permissions
|
1486
|
+
gapi.permissions.freeze
|
1487
|
+
end
|
1488
|
+
|
1277
1489
|
##
|
1278
1490
|
# Retrieves data from the table.
|
1279
1491
|
#
|
@@ -2162,8 +2374,12 @@ module Google
|
|
2162
2374
|
end
|
2163
2375
|
|
2164
2376
|
ensure_service!
|
2165
|
-
|
2166
|
-
|
2377
|
+
gapi = service.insert_tabledata dataset_id,
|
2378
|
+
table_id,
|
2379
|
+
rows,
|
2380
|
+
skip_invalid: skip_invalid,
|
2381
|
+
ignore_unknown: ignore_unknown,
|
2382
|
+
insert_ids: insert_ids
|
2167
2383
|
InsertResponse.from_gapi rows, gapi
|
2168
2384
|
end
|
2169
2385
|
|
@@ -2462,7 +2678,7 @@ module Google
|
|
2462
2678
|
return if attributes.empty?
|
2463
2679
|
ensure_service!
|
2464
2680
|
patch_args = Hash[attributes.map { |attr| [attr, @gapi.send(attr)] }]
|
2465
|
-
patch_gapi = Google::Apis::BigqueryV2::Table.new
|
2681
|
+
patch_gapi = Google::Apis::BigqueryV2::Table.new(**patch_args)
|
2466
2682
|
patch_gapi.etag = etag if etag
|
2467
2683
|
@gapi = service.patch_table dataset_id, table_id, patch_gapi
|
2468
2684
|
|
@@ -2588,12 +2804,11 @@ module Google
|
|
2588
2804
|
|
2589
2805
|
def load_local_or_uri file, updater
|
2590
2806
|
job_gapi = updater.to_gapi
|
2591
|
-
|
2592
|
-
|
2593
|
-
|
2594
|
-
|
2595
|
-
|
2596
|
-
job
|
2807
|
+
if local_file? file
|
2808
|
+
load_local file, job_gapi
|
2809
|
+
else
|
2810
|
+
load_storage file, job_gapi
|
2811
|
+
end
|
2597
2812
|
end
|
2598
2813
|
|
2599
2814
|
def storage_url? files
|
@@ -2646,6 +2861,7 @@ module Google
|
|
2646
2861
|
##
|
2647
2862
|
# @private Create an Updater object.
|
2648
2863
|
def initialize gapi
|
2864
|
+
super()
|
2649
2865
|
@updates = []
|
2650
2866
|
@gapi = gapi
|
2651
2867
|
@schema = nil
|