google-cloud-bigquery 1.24.0 → 1.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +52 -0
- data/CONTRIBUTING.md +2 -2
- data/LOGGING.md +1 -1
- data/lib/google/cloud/bigquery/convert.rb +0 -4
- data/lib/google/cloud/bigquery/copy_job.rb +1 -0
- data/lib/google/cloud/bigquery/data.rb +2 -2
- data/lib/google/cloud/bigquery/dataset.rb +106 -21
- data/lib/google/cloud/bigquery/dataset/access.rb +112 -14
- data/lib/google/cloud/bigquery/dataset/list.rb +2 -2
- data/lib/google/cloud/bigquery/external.rb +328 -3
- data/lib/google/cloud/bigquery/extract_job.rb +8 -10
- data/lib/google/cloud/bigquery/job.rb +43 -3
- data/lib/google/cloud/bigquery/job/list.rb +4 -4
- data/lib/google/cloud/bigquery/load_job.rb +177 -24
- data/lib/google/cloud/bigquery/model/list.rb +2 -2
- data/lib/google/cloud/bigquery/policy.rb +432 -0
- data/lib/google/cloud/bigquery/project.rb +3 -3
- data/lib/google/cloud/bigquery/project/list.rb +2 -2
- data/lib/google/cloud/bigquery/query_job.rb +25 -14
- data/lib/google/cloud/bigquery/routine.rb +128 -9
- data/lib/google/cloud/bigquery/routine/list.rb +2 -2
- data/lib/google/cloud/bigquery/service.rb +44 -13
- data/lib/google/cloud/bigquery/standard_sql.rb +4 -3
- data/lib/google/cloud/bigquery/table.rb +261 -45
- data/lib/google/cloud/bigquery/table/async_inserter.rb +24 -15
- data/lib/google/cloud/bigquery/table/list.rb +2 -2
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +16 -15
@@ -56,7 +56,8 @@ module Google
|
|
56
56
|
# @private The Service object.
|
57
57
|
attr_accessor :service
|
58
58
|
|
59
|
-
attr_reader :name
|
59
|
+
attr_reader :name
|
60
|
+
attr_reader :numeric_id
|
60
61
|
|
61
62
|
##
|
62
63
|
# Creates a new Service instance.
|
@@ -981,8 +982,7 @@ module Google
|
|
981
982
|
# @param [String] description A user-friendly description of the
|
982
983
|
# dataset.
|
983
984
|
# @param [Integer] expiration The default lifetime of all tables in the
|
984
|
-
# dataset, in milliseconds. The minimum value is
|
985
|
-
# (one hour).
|
985
|
+
# dataset, in milliseconds. The minimum value is `3_600_000` (one hour).
|
986
986
|
# @param [String] location The geographic location where the dataset
|
987
987
|
# should reside. Possible values include `EU` and `US`. The default
|
988
988
|
# value is `US`.
|
@@ -121,12 +121,12 @@ module Google
|
|
121
121
|
# puts project.name
|
122
122
|
# end
|
123
123
|
#
|
124
|
-
def all request_limit: nil
|
124
|
+
def all request_limit: nil, &block
|
125
125
|
request_limit = request_limit.to_i if request_limit
|
126
126
|
return enum_for :all, request_limit: request_limit unless block_given?
|
127
127
|
results = self
|
128
128
|
loop do
|
129
|
-
results.each
|
129
|
+
results.each(&block)
|
130
130
|
if request_limit
|
131
131
|
request_limit -= 1
|
132
132
|
break if request_limit.negative?
|
@@ -94,8 +94,7 @@ module Google
|
|
94
94
|
# otherwise.
|
95
95
|
#
|
96
96
|
def batch?
|
97
|
-
|
98
|
-
val == "BATCH"
|
97
|
+
@gapi.configuration.query.priority == "BATCH"
|
99
98
|
end
|
100
99
|
|
101
100
|
##
|
@@ -693,8 +692,11 @@ module Google
|
|
693
692
|
end
|
694
693
|
ensure_schema!
|
695
694
|
|
696
|
-
|
697
|
-
|
695
|
+
data_hash = service.list_tabledata destination_table_dataset_id,
|
696
|
+
destination_table_table_id,
|
697
|
+
token: token,
|
698
|
+
max: max,
|
699
|
+
start: start
|
698
700
|
Data.from_gapi_json data_hash, destination_table_gapi, @gapi, service
|
699
701
|
end
|
700
702
|
alias query_results data
|
@@ -705,12 +707,11 @@ module Google
|
|
705
707
|
##
|
706
708
|
# @private Create an Updater object.
|
707
709
|
def initialize service, gapi
|
710
|
+
super()
|
708
711
|
@service = service
|
709
712
|
@gapi = gapi
|
710
713
|
end
|
711
714
|
|
712
|
-
# rubocop:disable all
|
713
|
-
|
714
715
|
##
|
715
716
|
# @private Create an Updater from an options hash.
|
716
717
|
#
|
@@ -749,8 +750,6 @@ module Google
|
|
749
750
|
updater
|
750
751
|
end
|
751
752
|
|
752
|
-
# rubocop:enable all
|
753
|
-
|
754
753
|
##
|
755
754
|
# Sets the geographic location where the job should run. Required
|
756
755
|
# except for US and EU.
|
@@ -936,13 +935,13 @@ module Google
|
|
936
935
|
raise ArgumentError, "types must use the same format as params" if types.class != params.class
|
937
936
|
|
938
937
|
case params
|
939
|
-
when Array
|
938
|
+
when Array
|
940
939
|
@gapi.configuration.query.use_legacy_sql = false
|
941
940
|
@gapi.configuration.query.parameter_mode = "POSITIONAL"
|
942
941
|
@gapi.configuration.query.query_parameters = params.zip(types).map do |param, type|
|
943
942
|
Convert.to_query_param param, type
|
944
943
|
end
|
945
|
-
when Hash
|
944
|
+
when Hash
|
946
945
|
@gapi.configuration.query.use_legacy_sql = false
|
947
946
|
@gapi.configuration.query.parameter_mode = "NAMED"
|
948
947
|
@gapi.configuration.query.query_parameters = params.map do |name, param|
|
@@ -1593,9 +1592,20 @@ module Google
|
|
1593
1592
|
# end
|
1594
1593
|
#
|
1595
1594
|
class Stage
|
1596
|
-
attr_reader :compute_ratio_avg
|
1597
|
-
|
1598
|
-
|
1595
|
+
attr_reader :compute_ratio_avg
|
1596
|
+
attr_reader :compute_ratio_max
|
1597
|
+
attr_reader :id
|
1598
|
+
attr_reader :name
|
1599
|
+
attr_reader :read_ratio_avg
|
1600
|
+
attr_reader :read_ratio_max
|
1601
|
+
attr_reader :records_read
|
1602
|
+
attr_reader :records_written
|
1603
|
+
attr_reader :status
|
1604
|
+
attr_reader :steps
|
1605
|
+
attr_reader :wait_ratio_avg
|
1606
|
+
attr_reader :wait_ratio_max
|
1607
|
+
attr_reader :write_ratio_avg
|
1608
|
+
attr_reader :write_ratio_max
|
1599
1609
|
|
1600
1610
|
##
|
1601
1611
|
# @private Creates a new Stage instance.
|
@@ -1658,7 +1668,8 @@ module Google
|
|
1658
1668
|
# end
|
1659
1669
|
#
|
1660
1670
|
class Step
|
1661
|
-
attr_reader :kind
|
1671
|
+
attr_reader :kind
|
1672
|
+
attr_reader :substeps
|
1662
1673
|
|
1663
1674
|
##
|
1664
1675
|
# @private Creates a new Stage instance.
|
@@ -603,6 +603,93 @@ module Google
|
|
603
603
|
update_gapi!
|
604
604
|
end
|
605
605
|
|
606
|
+
###
|
607
|
+
# The JavaScript UDF determinism level. Optional.
|
608
|
+
#
|
609
|
+
# * `DETERMINISTIC` - Deterministic indicates that two calls with the same input to a UDF yield the same output.
|
610
|
+
# If all JavaScript UDFs are `DETERMINISTIC`, the query result is potentially cachable.
|
611
|
+
# * `NOT_DETERMINISTIC` - Not deterministic indicates that the output of the UDF is not guaranteed to yield the
|
612
|
+
# same output each time for a given set of inputs. If any JavaScript UDF is `NOT_DETERMINISTIC`, the query
|
613
|
+
# result is not cacheable.
|
614
|
+
#
|
615
|
+
# Even if a JavaScript UDF is deterministic, many other factors can prevent usage of cached query results.
|
616
|
+
# Example factors include but not limited to: DDL/DML, non-deterministic SQL function calls, update of
|
617
|
+
# referenced tables/views/UDFs or imported JavaScript libraries. SQL UDFs cannot have determinism specified.
|
618
|
+
# Their determinism is automatically determined.
|
619
|
+
#
|
620
|
+
# @return [String, nil] The routine determinism level in upper case, or `nil` if not set or the object is a
|
621
|
+
# reference (see {#reference?}).
|
622
|
+
#
|
623
|
+
# @example
|
624
|
+
# require "google/cloud/bigquery"
|
625
|
+
#
|
626
|
+
# bigquery = Google::Cloud::Bigquery.new
|
627
|
+
# dataset = bigquery.dataset "my_dataset"
|
628
|
+
# routine = dataset.routine "my_routine"
|
629
|
+
#
|
630
|
+
# routine.determinism_level #=> "NOT_DETERMINISTIC"
|
631
|
+
#
|
632
|
+
# @!group Attributes
|
633
|
+
#
|
634
|
+
def determinism_level
|
635
|
+
return nil if reference?
|
636
|
+
ensure_full_data!
|
637
|
+
@gapi.determinism_level
|
638
|
+
end
|
639
|
+
|
640
|
+
##
|
641
|
+
# Updates the JavaScript UDF determinism level. Optional.
|
642
|
+
#
|
643
|
+
# * `DETERMINISTIC` - Deterministic indicates that two calls with the same input to a UDF yield the same output.
|
644
|
+
# If all JavaScript UDFs are `DETERMINISTIC`, the query result is potentially cachable.
|
645
|
+
# * `NOT_DETERMINISTIC` - Not deterministic indicates that the output of the UDF is not guaranteed to yield the
|
646
|
+
# same output each time for a given set of inputs. If any JavaScript UDF is `NOT_DETERMINISTIC`, the query
|
647
|
+
# result is not cacheable.
|
648
|
+
#
|
649
|
+
# @param [String, nil] new_determinism_level The new routine determinism level in upper case.
|
650
|
+
#
|
651
|
+
# @example
|
652
|
+
# require "google/cloud/bigquery"
|
653
|
+
#
|
654
|
+
# bigquery = Google::Cloud::Bigquery.new
|
655
|
+
# dataset = bigquery.dataset "my_dataset"
|
656
|
+
# routine = dataset.routine "my_routine"
|
657
|
+
#
|
658
|
+
# routine.determinism_level #=> "NOT_DETERMINISTIC"
|
659
|
+
# routine.determinism_level = "DETERMINISTIC"
|
660
|
+
#
|
661
|
+
# @!group Attributes
|
662
|
+
#
|
663
|
+
def determinism_level= new_determinism_level
|
664
|
+
ensure_full_data!
|
665
|
+
@gapi.determinism_level = new_determinism_level
|
666
|
+
update_gapi!
|
667
|
+
end
|
668
|
+
|
669
|
+
##
|
670
|
+
# Checks if the value of {#determinism_level} is `DETERMINISTIC`. The default is `false`.
|
671
|
+
#
|
672
|
+
# @return [Boolean] `true` when `DETERMINISTIC` and the object is not a reference (see {#reference?}), `false`
|
673
|
+
# otherwise.
|
674
|
+
#
|
675
|
+
# @!group Attributes
|
676
|
+
#
|
677
|
+
def determinism_level_deterministic?
|
678
|
+
@gapi.determinism_level == "DETERMINISTIC"
|
679
|
+
end
|
680
|
+
|
681
|
+
##
|
682
|
+
# Checks if the value of {#determinism_level} is `NOT_DETERMINISTIC`. The default is `false`.
|
683
|
+
#
|
684
|
+
# @return [Boolean] `true` when `NOT_DETERMINISTIC` and the object is not a reference (see {#reference?}),
|
685
|
+
# `false` otherwise.
|
686
|
+
#
|
687
|
+
# @!group Attributes
|
688
|
+
#
|
689
|
+
def determinism_level_not_deterministic?
|
690
|
+
@gapi.determinism_level == "NOT_DETERMINISTIC"
|
691
|
+
end
|
692
|
+
|
606
693
|
##
|
607
694
|
# Updates the routine with changes made in the given block in a single update request. The following attributes
|
608
695
|
# may be set: {Updater#routine_type=}, {Updater#language=}, {Updater#arguments=}, {Updater#return_type=},
|
@@ -919,6 +1006,7 @@ module Google
|
|
919
1006
|
##
|
920
1007
|
# @private Create an Updater object.
|
921
1008
|
def initialize gapi
|
1009
|
+
super()
|
922
1010
|
@original_gapi = gapi
|
923
1011
|
@gapi = gapi.dup
|
924
1012
|
end
|
@@ -999,7 +1087,9 @@ module Google
|
|
999
1087
|
# routine = dataset.routine "my_routine"
|
1000
1088
|
#
|
1001
1089
|
# routine.return_type.type_kind #=> "INT64"
|
1002
|
-
# routine.
|
1090
|
+
# routine.update do |r|
|
1091
|
+
# r.return_type = "STRING"
|
1092
|
+
# end
|
1003
1093
|
#
|
1004
1094
|
def return_type= new_return_type
|
1005
1095
|
@gapi.return_type = StandardSql::DataType.gapi_from_string_or_data_type new_return_type
|
@@ -1019,9 +1109,11 @@ module Google
|
|
1019
1109
|
# dataset = bigquery.dataset "my_dataset"
|
1020
1110
|
# routine = dataset.routine "my_routine"
|
1021
1111
|
#
|
1022
|
-
# routine.
|
1023
|
-
#
|
1024
|
-
#
|
1112
|
+
# routine.update do |r|
|
1113
|
+
# r.imported_libraries = [
|
1114
|
+
# "gs://cloud-samples-data/bigquery/udfs/max-value.js"
|
1115
|
+
# ]
|
1116
|
+
# end
|
1025
1117
|
#
|
1026
1118
|
def imported_libraries= new_imported_libraries
|
1027
1119
|
@gapi.imported_libraries = new_imported_libraries
|
@@ -1069,12 +1161,43 @@ module Google
|
|
1069
1161
|
# routine = dataset.routine "my_routine"
|
1070
1162
|
#
|
1071
1163
|
# routine.description #=> "My routine description"
|
1072
|
-
# routine.
|
1164
|
+
# routine.update do |r|
|
1165
|
+
# r.description = "My updated routine description"
|
1166
|
+
# end
|
1073
1167
|
#
|
1074
1168
|
def description= new_description
|
1075
1169
|
@gapi.description = new_description
|
1076
1170
|
end
|
1077
1171
|
|
1172
|
+
##
|
1173
|
+
# Updates the JavaScript UDF determinism level. Optional.
|
1174
|
+
#
|
1175
|
+
# * `DETERMINISTIC` - Deterministic indicates that two calls with the same input to a UDF yield the same
|
1176
|
+
# output. If all JavaScript UDFs are `DETERMINISTIC`, the query result is potentially cachable.
|
1177
|
+
# * `NOT_DETERMINISTIC` - Not deterministic indicates that the output of the UDF is not guaranteed to yield
|
1178
|
+
# the same output each time for a given set of inputs. If any JavaScript UDF is `NOT_DETERMINISTIC`, the
|
1179
|
+
# query result is not cacheable.
|
1180
|
+
#
|
1181
|
+
# @param [String, nil] new_determinism_level The new routine determinism level in upper case.
|
1182
|
+
#
|
1183
|
+
# @example
|
1184
|
+
# require "google/cloud/bigquery"
|
1185
|
+
#
|
1186
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1187
|
+
# dataset = bigquery.dataset "my_dataset"
|
1188
|
+
# routine = dataset.routine "my_routine"
|
1189
|
+
#
|
1190
|
+
# routine.determinism_level #=> "NOT_DETERMINISTIC"
|
1191
|
+
# routine.update do |r|
|
1192
|
+
# r.determinism_level = "DETERMINISTIC"
|
1193
|
+
# end
|
1194
|
+
#
|
1195
|
+
# @!group Attributes
|
1196
|
+
#
|
1197
|
+
def determinism_level= new_determinism_level
|
1198
|
+
@gapi.determinism_level = new_determinism_level
|
1199
|
+
end
|
1200
|
+
|
1078
1201
|
def update
|
1079
1202
|
raise "not implemented in #{self.class}"
|
1080
1203
|
end
|
@@ -1088,15 +1211,11 @@ module Google
|
|
1088
1211
|
end
|
1089
1212
|
alias refresh! reload!
|
1090
1213
|
|
1091
|
-
# rubocop:disable Style/CaseEquality
|
1092
|
-
|
1093
1214
|
# @private
|
1094
1215
|
def updates?
|
1095
1216
|
!(@gapi === @original_gapi)
|
1096
1217
|
end
|
1097
1218
|
|
1098
|
-
# rubocop:enable Style/CaseEquality
|
1099
|
-
|
1100
1219
|
# @private
|
1101
1220
|
def to_gapi
|
1102
1221
|
@gapi
|
@@ -124,12 +124,12 @@ module Google
|
|
124
124
|
# puts routine.routine_id
|
125
125
|
# end
|
126
126
|
#
|
127
|
-
def all request_limit: nil
|
127
|
+
def all request_limit: nil, &block
|
128
128
|
request_limit = request_limit.to_i if request_limit
|
129
129
|
return enum_for :all, request_limit: request_limit unless block_given?
|
130
130
|
results = self
|
131
131
|
loop do
|
132
|
-
results.each
|
132
|
+
results.each(&block)
|
133
133
|
if request_limit
|
134
134
|
request_limit -= 1
|
135
135
|
break if request_limit.negative?
|
@@ -182,6 +182,34 @@ module Google
|
|
182
182
|
end
|
183
183
|
end
|
184
184
|
|
185
|
+
##
|
186
|
+
# Returns Google::Apis::BigqueryV2::Policy
|
187
|
+
def get_table_policy dataset_id, table_id
|
188
|
+
policy_options = API::GetPolicyOptions.new requested_policy_version: 1
|
189
|
+
execute do
|
190
|
+
service.get_table_iam_policy table_path(dataset_id, table_id),
|
191
|
+
API::GetIamPolicyRequest.new(options: policy_options)
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
##
|
196
|
+
# @param [Google::Apis::BigqueryV2::Policy] new_policy
|
197
|
+
def set_table_policy dataset_id, table_id, new_policy
|
198
|
+
execute do
|
199
|
+
service.set_table_iam_policy table_path(dataset_id, table_id),
|
200
|
+
API::SetIamPolicyRequest.new(policy: new_policy)
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
##
|
205
|
+
# Returns Google::Apis::BigqueryV2::TestIamPermissionsResponse
|
206
|
+
def test_table_permissions dataset_id, table_id, permissions
|
207
|
+
execute do
|
208
|
+
service.test_table_iam_permissions table_path(dataset_id, table_id),
|
209
|
+
API::TestIamPermissionsRequest.new(permissions: permissions)
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
185
213
|
##
|
186
214
|
# Deletes the table specified by tableId from the dataset.
|
187
215
|
# If the table contains data, all the data will be deleted.
|
@@ -455,7 +483,7 @@ module Google
|
|
455
483
|
table_id: m["tbl"]
|
456
484
|
}.delete_if { |_, v| v.nil? }
|
457
485
|
str_table_ref_hash = default_ref.to_h.merge str_table_ref_hash
|
458
|
-
ref = Google::Apis::BigqueryV2::TableReference.new
|
486
|
+
ref = Google::Apis::BigqueryV2::TableReference.new(**str_table_ref_hash)
|
459
487
|
validate_table_ref ref
|
460
488
|
ref
|
461
489
|
end
|
@@ -508,6 +536,11 @@ module Google
|
|
508
536
|
|
509
537
|
protected
|
510
538
|
|
539
|
+
# Creates a formatted table path.
|
540
|
+
def table_path dataset_id, table_id
|
541
|
+
"projects/#{@project}/datasets/#{dataset_id}/tables/#{table_id}"
|
542
|
+
end
|
543
|
+
|
511
544
|
# Generate a random string similar to the BigQuery service job IDs.
|
512
545
|
def generate_id
|
513
546
|
SecureRandom.urlsafe_base64 21
|
@@ -521,9 +554,9 @@ module Google
|
|
521
554
|
nil
|
522
555
|
end
|
523
556
|
|
524
|
-
def execute backoff: nil
|
557
|
+
def execute backoff: nil, &block
|
525
558
|
if backoff
|
526
|
-
Backoff.new(retries: retries).execute
|
559
|
+
Backoff.new(retries: retries).execute(&block)
|
527
560
|
else
|
528
561
|
yield
|
529
562
|
end
|
@@ -557,22 +590,20 @@ module Google
|
|
557
590
|
def execute
|
558
591
|
current_retries = 0
|
559
592
|
loop do
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
564
|
-
|
565
|
-
|
566
|
-
current_retries += 1
|
567
|
-
end
|
593
|
+
return yield
|
594
|
+
rescue Google::Apis::Error => e
|
595
|
+
raise e unless retry? e.body, current_retries
|
596
|
+
|
597
|
+
@backoff.call current_retries
|
598
|
+
current_retries += 1
|
568
599
|
end
|
569
600
|
end
|
570
601
|
|
571
602
|
protected
|
572
603
|
|
573
604
|
def retry? result, current_retries #:nodoc:
|
574
|
-
if current_retries < @retries
|
575
|
-
return true
|
605
|
+
if current_retries < @retries && retry_error_reason?(result)
|
606
|
+
return true
|
576
607
|
end
|
577
608
|
false
|
578
609
|
end
|
@@ -401,11 +401,12 @@ module Google
|
|
401
401
|
# @private New Google::Apis::BigqueryV2::StandardSqlDataType from a String or StandardSql::DataType object.
|
402
402
|
def self.gapi_from_string_or_data_type data_type
|
403
403
|
return if data_type.nil?
|
404
|
-
|
404
|
+
case data_type
|
405
|
+
when StandardSql::DataType
|
405
406
|
data_type.to_gapi
|
406
|
-
|
407
|
+
when Hash
|
407
408
|
data_type
|
408
|
-
|
409
|
+
when String, Symbol
|
409
410
|
Google::Apis::BigqueryV2::StandardSqlDataType.new type_kind: data_type.to_s.upcase
|
410
411
|
else
|
411
412
|
raise ArgumentError, "Unable to convert #{data_type} to Google::Apis::BigqueryV2::StandardSqlDataType"
|
@@ -23,6 +23,7 @@ require "google/cloud/bigquery/external"
|
|
23
23
|
require "google/cloud/bigquery/insert_response"
|
24
24
|
require "google/cloud/bigquery/table/async_inserter"
|
25
25
|
require "google/cloud/bigquery/convert"
|
26
|
+
require "google/cloud/bigquery/policy"
|
26
27
|
require "google/apis/bigquery_v2"
|
27
28
|
|
28
29
|
module Google
|
@@ -36,16 +37,16 @@ module Google
|
|
36
37
|
# repeated fields.
|
37
38
|
#
|
38
39
|
# The Table class can also represent a
|
39
|
-
# [view](https://cloud.google.com/bigquery/docs/views), which is a virtual
|
40
|
-
# table defined by a SQL query
|
41
|
-
# materialized views,
|
42
|
-
#
|
43
|
-
#
|
44
|
-
# indirectly by the top-level query. (See {#view?}, {#query}, {#query=},
|
45
|
-
# and {Dataset#create_view}.)
|
40
|
+
# [logical view](https://cloud.google.com/bigquery/docs/views), which is a virtual
|
41
|
+
# table defined by a SQL query (see {#view?} and {Dataset#create_view}); or a
|
42
|
+
# [materialized view](https://cloud.google.com/bigquery/docs/materialized-views-intro),
|
43
|
+
# which is a precomputed view that periodically caches results of a query for increased
|
44
|
+
# performance and efficiency (see {#materialized_view?} and {Dataset#create_materialized_view}).
|
46
45
|
#
|
47
46
|
# @see https://cloud.google.com/bigquery/docs/loading-data#loading_denormalized_nested_and_repeated_data
|
48
47
|
# Loading denormalized, nested, and repeated data
|
48
|
+
# @see https://cloud.google.com/bigquery/docs/views Creating views
|
49
|
+
# @see https://cloud.google.com/bigquery/docs/materialized-views-intro Introduction to materialized views
|
49
50
|
#
|
50
51
|
# @example
|
51
52
|
# require "google/cloud/bigquery"
|
@@ -76,7 +77,7 @@ module Google
|
|
76
77
|
# }
|
77
78
|
# table.insert row
|
78
79
|
#
|
79
|
-
# @example Creating a
|
80
|
+
# @example Creating a logical view:
|
80
81
|
# require "google/cloud/bigquery"
|
81
82
|
#
|
82
83
|
# bigquery = Google::Cloud::Bigquery.new
|
@@ -85,6 +86,15 @@ module Google
|
|
85
86
|
# "SELECT name, age FROM `my_project.my_dataset.my_table`"
|
86
87
|
# view.view? # true
|
87
88
|
#
|
89
|
+
# @example Creating a materialized view:
|
90
|
+
# require "google/cloud/bigquery"
|
91
|
+
#
|
92
|
+
# bigquery = Google::Cloud::Bigquery.new
|
93
|
+
# dataset = bigquery.dataset "my_dataset"
|
94
|
+
# view = dataset.create_materialized_view "my_materialized_view",
|
95
|
+
# "SELECT name, age FROM `my_project.my_dataset.my_table`"
|
96
|
+
# view.materialized_view? # true
|
97
|
+
#
|
88
98
|
class Table
|
89
99
|
##
|
90
100
|
# @private The Service object.
|
@@ -725,7 +735,7 @@ module Google
|
|
725
735
|
end
|
726
736
|
|
727
737
|
##
|
728
|
-
# Checks if the table's type is
|
738
|
+
# Checks if the table's type is `TABLE`.
|
729
739
|
#
|
730
740
|
# @return [Boolean, nil] `true` when the type is `TABLE`, `false`
|
731
741
|
# otherwise, if the object is a resource (see {#resource?}); `nil` if
|
@@ -739,8 +749,10 @@ module Google
|
|
739
749
|
end
|
740
750
|
|
741
751
|
##
|
742
|
-
# Checks if the table's type is
|
743
|
-
# represents a BigQuery view. See {Dataset#create_view}.
|
752
|
+
# Checks if the table's type is `VIEW`, indicating that the table
|
753
|
+
# represents a BigQuery logical view. See {Dataset#create_view}.
|
754
|
+
#
|
755
|
+
# @see https://cloud.google.com/bigquery/docs/views Creating views
|
744
756
|
#
|
745
757
|
# @return [Boolean, nil] `true` when the type is `VIEW`, `false`
|
746
758
|
# otherwise, if the object is a resource (see {#resource?}); `nil` if
|
@@ -754,7 +766,25 @@ module Google
|
|
754
766
|
end
|
755
767
|
|
756
768
|
##
|
757
|
-
# Checks if the table's type is
|
769
|
+
# Checks if the table's type is `MATERIALIZED_VIEW`, indicating that
|
770
|
+
# the table represents a BigQuery materialized view.
|
771
|
+
# See {Dataset#create_materialized_view}.
|
772
|
+
#
|
773
|
+
# @see https://cloud.google.com/bigquery/docs/materialized-views-intro Introduction to materialized views
|
774
|
+
#
|
775
|
+
# @return [Boolean, nil] `true` when the type is `MATERIALIZED_VIEW`,
|
776
|
+
# `false` otherwise, if the object is a resource (see {#resource?});
|
777
|
+
# `nil` if the object is a reference (see {#reference?}).
|
778
|
+
#
|
779
|
+
# @!group Attributes
|
780
|
+
#
|
781
|
+
def materialized_view?
|
782
|
+
return nil if reference?
|
783
|
+
@gapi.type == "MATERIALIZED_VIEW"
|
784
|
+
end
|
785
|
+
|
786
|
+
##
|
787
|
+
# Checks if the table's type is `EXTERNAL`, indicating that the table
|
758
788
|
# represents an External Data Source. See {#external?} and
|
759
789
|
# {External::DataSource}.
|
760
790
|
#
|
@@ -1137,21 +1167,24 @@ module Google
|
|
1137
1167
|
end
|
1138
1168
|
|
1139
1169
|
##
|
1140
|
-
# The query that
|
1170
|
+
# The query that defines the view or materialized view. See {#view?} and
|
1171
|
+
# {#materialized_view?}.
|
1141
1172
|
#
|
1142
|
-
# @return [String] The query that defines the view
|
1173
|
+
# @return [String, nil] The query that defines the view or materialized_view;
|
1174
|
+
# or `nil` if not a view or materialized view.
|
1143
1175
|
#
|
1144
1176
|
# @!group Attributes
|
1145
1177
|
#
|
1146
1178
|
def query
|
1147
|
-
@gapi.view&.query
|
1179
|
+
view? ? @gapi.view&.query : @gapi.materialized_view&.query
|
1148
1180
|
end
|
1149
1181
|
|
1150
1182
|
##
|
1151
|
-
# Updates the query that
|
1183
|
+
# Updates the query that defines the view. (See {#view?}.) Not supported
|
1184
|
+
# for materialized views.
|
1152
1185
|
#
|
1153
|
-
# This sets the query using standard SQL. To specify legacy SQL or
|
1154
|
-
# use user-defined function resources use (#set_query) instead.
|
1186
|
+
# This method sets the query using standard SQL. To specify legacy SQL or
|
1187
|
+
# to use user-defined function resources for a view, use (#set_query) instead.
|
1155
1188
|
#
|
1156
1189
|
# @see https://cloud.google.com/bigquery/query-reference BigQuery Query
|
1157
1190
|
# Reference
|
@@ -1166,7 +1199,7 @@ module Google
|
|
1166
1199
|
# view = dataset.table "my_view"
|
1167
1200
|
#
|
1168
1201
|
# view.query = "SELECT first_name FROM " \
|
1169
|
-
#
|
1202
|
+
# "`my_project.my_dataset.my_table`"
|
1170
1203
|
#
|
1171
1204
|
# @!group Lifecycle
|
1172
1205
|
#
|
@@ -1175,12 +1208,12 @@ module Google
|
|
1175
1208
|
end
|
1176
1209
|
|
1177
1210
|
##
|
1178
|
-
# Updates the query that
|
1179
|
-
#
|
1180
|
-
# resources.
|
1211
|
+
# Updates the query that defines the view. (See {#view?}.) Not supported for
|
1212
|
+
# materialized views.
|
1181
1213
|
#
|
1182
|
-
#
|
1183
|
-
#
|
1214
|
+
# Allows setting of standard vs. legacy SQL and user-defined function resources.
|
1215
|
+
#
|
1216
|
+
# @see https://cloud.google.com/bigquery/query-reference BigQuery Query Reference
|
1184
1217
|
#
|
1185
1218
|
# @param [String] query The query that defines the view.
|
1186
1219
|
# @param [Boolean] standard_sql Specifies whether to use BigQuery's
|
@@ -1192,11 +1225,12 @@ module Google
|
|
1192
1225
|
# SQL](https://cloud.google.com/bigquery/docs/reference/legacy-sql)
|
1193
1226
|
# dialect. Optional. The default value is false.
|
1194
1227
|
# @param [Array<String>, String] udfs User-defined function resources
|
1195
|
-
# used in a legacy SQL query.
|
1196
|
-
#
|
1197
|
-
#
|
1198
|
-
#
|
1199
|
-
#
|
1228
|
+
# used in a legacy SQL query. Optional.
|
1229
|
+
#
|
1230
|
+
# May be either a code resource to load from a Google Cloud Storage URI
|
1231
|
+
# (`gs://bucket/path`), or an inline resource that contains code for a
|
1232
|
+
# user-defined function (UDF). Providing an inline code resource is equivalent
|
1233
|
+
# to providing a URI for a file containing the same code.
|
1200
1234
|
#
|
1201
1235
|
# This parameter is used for defining User Defined Function (UDF)
|
1202
1236
|
# resources only when using legacy SQL. Users of standard SQL should
|
@@ -1207,7 +1241,7 @@ module Google
|
|
1207
1241
|
# standard SQL - Differences in user-defined JavaScript
|
1208
1242
|
# functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/migrating-from-legacy-sql#differences_in_user-defined_javascript_functions)
|
1209
1243
|
#
|
1210
|
-
# @example
|
1244
|
+
# @example Update a view:
|
1211
1245
|
# require "google/cloud/bigquery"
|
1212
1246
|
#
|
1213
1247
|
# bigquery = Google::Cloud::Bigquery.new
|
@@ -1215,12 +1249,13 @@ module Google
|
|
1215
1249
|
# view = dataset.table "my_view"
|
1216
1250
|
#
|
1217
1251
|
# view.set_query "SELECT first_name FROM " \
|
1218
|
-
#
|
1252
|
+
# "`my_project.my_dataset.my_table`",
|
1219
1253
|
# standard_sql: true
|
1220
1254
|
#
|
1221
1255
|
# @!group Lifecycle
|
1222
1256
|
#
|
1223
1257
|
def set_query query, standard_sql: nil, legacy_sql: nil, udfs: nil
|
1258
|
+
raise "Updating the query is not supported for Table type: #{@gapi.type}" unless view?
|
1224
1259
|
use_legacy_sql = Convert.resolve_legacy_sql standard_sql, legacy_sql
|
1225
1260
|
@gapi.view = Google::Apis::BigqueryV2::ViewDefinition.new(
|
1226
1261
|
query: query,
|
@@ -1231,26 +1266,28 @@ module Google
|
|
1231
1266
|
end
|
1232
1267
|
|
1233
1268
|
##
|
1234
|
-
# Checks if the view's query is using legacy sql.
|
1269
|
+
# Checks if the view's query is using legacy sql. See {#view?}.
|
1235
1270
|
#
|
1236
|
-
# @return [Boolean] `true` when legacy sql is used, `false` otherwise.
|
1271
|
+
# @return [Boolean] `true` when legacy sql is used, `false` otherwise; or `nil` if not a logical view.
|
1237
1272
|
#
|
1238
1273
|
# @!group Attributes
|
1239
1274
|
#
|
1240
1275
|
def query_legacy_sql?
|
1276
|
+
return nil unless @gapi.view
|
1241
1277
|
val = @gapi.view.use_legacy_sql
|
1242
1278
|
return true if val.nil?
|
1243
1279
|
val
|
1244
1280
|
end
|
1245
1281
|
|
1246
1282
|
##
|
1247
|
-
# Checks if the view's query is using standard sql.
|
1283
|
+
# Checks if the view's query is using standard sql. See {#view?}.
|
1248
1284
|
#
|
1249
1285
|
# @return [Boolean] `true` when standard sql is used, `false` otherwise.
|
1250
1286
|
#
|
1251
1287
|
# @!group Attributes
|
1252
1288
|
#
|
1253
1289
|
def query_standard_sql?
|
1290
|
+
return nil unless @gapi.view
|
1254
1291
|
!query_legacy_sql?
|
1255
1292
|
end
|
1256
1293
|
|
@@ -1262,18 +1299,193 @@ module Google
|
|
1262
1299
|
# equivalent to providing a URI for a file containing the same code. See
|
1263
1300
|
# [User-Defined
|
1264
1301
|
# Functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/user-defined-functions).
|
1302
|
+
# See {#view?}.
|
1265
1303
|
#
|
1266
|
-
# @return [Array<String
|
1267
|
-
# and/or inline source code.
|
1304
|
+
# @return [Array<String>, nil] An array containing Google Cloud Storage URIs
|
1305
|
+
# and/or inline source code, or `nil` if not a logical view.
|
1268
1306
|
#
|
1269
1307
|
# @!group Attributes
|
1270
1308
|
#
|
1271
1309
|
def query_udfs
|
1310
|
+
return nil unless @gapi.view
|
1272
1311
|
udfs_gapi = @gapi.view.user_defined_function_resources
|
1273
1312
|
return [] if udfs_gapi.nil?
|
1274
1313
|
Array(udfs_gapi).map { |udf| udf.inline_code || udf.resource_uri }
|
1275
1314
|
end
|
1276
1315
|
|
1316
|
+
##
|
1317
|
+
# Whether automatic refresh of the materialized view is enabled. When true,
|
1318
|
+
# the materialized view is updated when the base table is updated. The default
|
1319
|
+
# value is true. See {#materialized_view?}.
|
1320
|
+
#
|
1321
|
+
# @return [Boolean, nil] `true` when automatic refresh is enabled, `false` otherwise;
|
1322
|
+
# or `nil` if not a materialized view.
|
1323
|
+
#
|
1324
|
+
# @!group Attributes
|
1325
|
+
#
|
1326
|
+
def enable_refresh?
|
1327
|
+
return nil unless @gapi.materialized_view
|
1328
|
+
val = @gapi.materialized_view.enable_refresh
|
1329
|
+
return true if val.nil?
|
1330
|
+
val
|
1331
|
+
end
|
1332
|
+
|
1333
|
+
##
|
1334
|
+
# Sets whether automatic refresh of the materialized view is enabled. When true,
|
1335
|
+
# the materialized view is updated when the base table is updated. See {#materialized_view?}.
|
1336
|
+
#
|
1337
|
+
# @param [Boolean] new_enable_refresh `true` when automatic refresh is enabled, `false` otherwise.
|
1338
|
+
#
|
1339
|
+
# @!group Attributes
|
1340
|
+
#
|
1341
|
+
def enable_refresh= new_enable_refresh
|
1342
|
+
@gapi.materialized_view = Google::Apis::BigqueryV2::MaterializedViewDefinition.new(
|
1343
|
+
enable_refresh: new_enable_refresh
|
1344
|
+
)
|
1345
|
+
patch_gapi! :materialized_view
|
1346
|
+
end
|
1347
|
+
|
1348
|
+
##
|
1349
|
+
# The time when the materialized view was last modified.
|
1350
|
+
# See {#materialized_view?}.
|
1351
|
+
#
|
1352
|
+
# @return [Time, nil] The time, or `nil` if not present or not a materialized view.
|
1353
|
+
#
|
1354
|
+
# @!group Attributes
|
1355
|
+
#
|
1356
|
+
def last_refresh_time
|
1357
|
+
Convert.millis_to_time @gapi.materialized_view&.last_refresh_time
|
1358
|
+
end
|
1359
|
+
|
1360
|
+
##
|
1361
|
+
# The maximum frequency in milliseconds at which the materialized view will be refreshed.
|
1362
|
+
# See {#materialized_view?}.
|
1363
|
+
#
|
1364
|
+
# @return [Integer, nil] The maximum frequency in milliseconds;
|
1365
|
+
# or `nil` if not a materialized view.
|
1366
|
+
#
|
1367
|
+
# @!group Attributes
|
1368
|
+
#
|
1369
|
+
def refresh_interval_ms
|
1370
|
+
@gapi.materialized_view&.refresh_interval_ms
|
1371
|
+
end
|
1372
|
+
|
1373
|
+
##
|
1374
|
+
# Sets the maximum frequency at which the materialized view will be refreshed.
|
1375
|
+
# See {#materialized_view?}.
|
1376
|
+
#
|
1377
|
+
# @param [Integer] new_refresh_interval_ms The maximum frequency in milliseconds.
|
1378
|
+
#
|
1379
|
+
# @!group Attributes
|
1380
|
+
#
|
1381
|
+
def refresh_interval_ms= new_refresh_interval_ms
|
1382
|
+
@gapi.materialized_view = Google::Apis::BigqueryV2::MaterializedViewDefinition.new(
|
1383
|
+
refresh_interval_ms: new_refresh_interval_ms
|
1384
|
+
)
|
1385
|
+
patch_gapi! :materialized_view
|
1386
|
+
end
|
1387
|
+
|
1388
|
+
##
|
1389
|
+
# Gets the Cloud IAM access control policy for the table. The latest policy will be read from the service. See
|
1390
|
+
# also {#update_policy}.
|
1391
|
+
#
|
1392
|
+
# @see https://cloud.google.com/iam/docs/managing-policies Managing Policies
|
1393
|
+
# @see https://cloud.google.com/bigquery/docs/table-access-controls-intro Controlling access to tables
|
1394
|
+
#
|
1395
|
+
# @return [Policy] The frozen policy for the table.
|
1396
|
+
#
|
1397
|
+
# @example
|
1398
|
+
# require "google/cloud/bigquery"
|
1399
|
+
#
|
1400
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1401
|
+
# dataset = bigquery.dataset "my_dataset"
|
1402
|
+
# table = dataset.table "my_table"
|
1403
|
+
#
|
1404
|
+
# policy = table.policy
|
1405
|
+
#
|
1406
|
+
# policy.frozen? #=> true
|
1407
|
+
# binding_owner = policy.bindings.find { |b| b.role == "roles/owner" }
|
1408
|
+
# binding_owner.role #=> "roles/owner"
|
1409
|
+
# binding_owner.members #=> ["user:owner@example.com"]
|
1410
|
+
# binding_owner.frozen? #=> true
|
1411
|
+
# binding_owner.members.frozen? #=> true
|
1412
|
+
#
|
1413
|
+
def policy
|
1414
|
+
raise ArgumentError, "Block argument not supported: Use #update_policy instead." if block_given?
|
1415
|
+
ensure_service!
|
1416
|
+
gapi = service.get_table_policy dataset_id, table_id
|
1417
|
+
Policy.from_gapi(gapi).freeze
|
1418
|
+
end
|
1419
|
+
|
1420
|
+
##
|
1421
|
+
# Updates the Cloud IAM access control policy for the table. The latest policy will be read from the service.
|
1422
|
+
# See also {#policy}.
|
1423
|
+
#
|
1424
|
+
# @see https://cloud.google.com/iam/docs/managing-policies Managing Policies
|
1425
|
+
# @see https://cloud.google.com/bigquery/docs/table-access-controls-intro Controlling access to tables
|
1426
|
+
#
|
1427
|
+
# @yield [policy] A block for updating the policy. The latest policy will be read from the service and passed to
|
1428
|
+
# the block. After the block completes, the modified policy will be written to the service.
|
1429
|
+
# @yieldparam [Policy] policy The mutable Policy for the table.
|
1430
|
+
#
|
1431
|
+
# @return [Policy] The updated and frozen policy for the table.
|
1432
|
+
#
|
1433
|
+
# @example Update the policy by passing a block.
|
1434
|
+
# require "google/cloud/bigquery"
|
1435
|
+
#
|
1436
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1437
|
+
# dataset = bigquery.dataset "my_dataset"
|
1438
|
+
# table = dataset.table "my_table"
|
1439
|
+
#
|
1440
|
+
# table.update_policy do |p|
|
1441
|
+
# p.grant role: "roles/viewer", members: "user:viewer@example.com"
|
1442
|
+
# p.revoke role: "roles/editor", members: "user:editor@example.com"
|
1443
|
+
# p.revoke role: "roles/owner"
|
1444
|
+
# end # 2 API calls
|
1445
|
+
#
|
1446
|
+
def update_policy
|
1447
|
+
raise ArgumentError, "A block updating the policy must be provided" unless block_given?
|
1448
|
+
ensure_service!
|
1449
|
+
gapi = service.get_table_policy dataset_id, table_id
|
1450
|
+
policy = Policy.from_gapi gapi
|
1451
|
+
yield policy
|
1452
|
+
# TODO: Check for changes before calling RPC
|
1453
|
+
gapi = service.set_table_policy dataset_id, table_id, policy.to_gapi
|
1454
|
+
Policy.from_gapi(gapi).freeze
|
1455
|
+
end
|
1456
|
+
|
1457
|
+
##
|
1458
|
+
# Tests the specified permissions against the [Cloud
|
1459
|
+
# IAM](https://cloud.google.com/iam/) access control policy.
|
1460
|
+
#
|
1461
|
+
# @see https://cloud.google.com/iam/docs/managing-policies Managing Policies
|
1462
|
+
#
|
1463
|
+
# @param [String, Array<String>] permissions The set of permissions
|
1464
|
+
# against which to check access. Permissions must be of the format
|
1465
|
+
# `bigquery.resource.capability`.
|
1466
|
+
# See https://cloud.google.com/bigquery/docs/access-control#bigquery.
|
1467
|
+
#
|
1468
|
+
# @return [Array<String>] The frozen array of permissions held by the caller.
|
1469
|
+
#
|
1470
|
+
# @example
|
1471
|
+
# require "google/cloud/bigquery"
|
1472
|
+
#
|
1473
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1474
|
+
# dataset = bigquery.dataset "my_dataset"
|
1475
|
+
# table = dataset.table "my_table"
|
1476
|
+
#
|
1477
|
+
# permissions = table.test_iam_permissions "bigquery.tables.get",
|
1478
|
+
# "bigquery.tables.delete"
|
1479
|
+
# permissions.include? "bigquery.tables.get" #=> true
|
1480
|
+
# permissions.include? "bigquery.tables.delete" #=> false
|
1481
|
+
#
|
1482
|
+
def test_iam_permissions *permissions
|
1483
|
+
permissions = Array(permissions).flatten
|
1484
|
+
ensure_service!
|
1485
|
+
gapi = service.test_table_permissions dataset_id, table_id, permissions
|
1486
|
+
gapi.permissions.freeze
|
1487
|
+
end
|
1488
|
+
|
1277
1489
|
##
|
1278
1490
|
# Retrieves data from the table.
|
1279
1491
|
#
|
@@ -2162,8 +2374,12 @@ module Google
|
|
2162
2374
|
end
|
2163
2375
|
|
2164
2376
|
ensure_service!
|
2165
|
-
|
2166
|
-
|
2377
|
+
gapi = service.insert_tabledata dataset_id,
|
2378
|
+
table_id,
|
2379
|
+
rows,
|
2380
|
+
skip_invalid: skip_invalid,
|
2381
|
+
ignore_unknown: ignore_unknown,
|
2382
|
+
insert_ids: insert_ids
|
2167
2383
|
InsertResponse.from_gapi rows, gapi
|
2168
2384
|
end
|
2169
2385
|
|
@@ -2462,7 +2678,7 @@ module Google
|
|
2462
2678
|
return if attributes.empty?
|
2463
2679
|
ensure_service!
|
2464
2680
|
patch_args = Hash[attributes.map { |attr| [attr, @gapi.send(attr)] }]
|
2465
|
-
patch_gapi = Google::Apis::BigqueryV2::Table.new
|
2681
|
+
patch_gapi = Google::Apis::BigqueryV2::Table.new(**patch_args)
|
2466
2682
|
patch_gapi.etag = etag if etag
|
2467
2683
|
@gapi = service.patch_table dataset_id, table_id, patch_gapi
|
2468
2684
|
|
@@ -2588,12 +2804,11 @@ module Google
|
|
2588
2804
|
|
2589
2805
|
def load_local_or_uri file, updater
|
2590
2806
|
job_gapi = updater.to_gapi
|
2591
|
-
|
2592
|
-
|
2593
|
-
|
2594
|
-
|
2595
|
-
|
2596
|
-
job
|
2807
|
+
if local_file? file
|
2808
|
+
load_local file, job_gapi
|
2809
|
+
else
|
2810
|
+
load_storage file, job_gapi
|
2811
|
+
end
|
2597
2812
|
end
|
2598
2813
|
|
2599
2814
|
def storage_url? files
|
@@ -2646,6 +2861,7 @@ module Google
|
|
2646
2861
|
##
|
2647
2862
|
# @private Create an Updater object.
|
2648
2863
|
def initialize gapi
|
2864
|
+
super()
|
2649
2865
|
@updates = []
|
2650
2866
|
@gapi = gapi
|
2651
2867
|
@schema = nil
|