google-cloud-bigquery 1.16.0 → 1.17.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/AUTHENTICATION.md +7 -27
- data/CHANGELOG.md +8 -0
- data/CONTRIBUTING.md +1 -1
- data/OVERVIEW.md +5 -5
- data/lib/google-cloud-bigquery.rb +5 -11
- data/lib/google/cloud/bigquery.rb +3 -5
- data/lib/google/cloud/bigquery/convert.rb +23 -46
- data/lib/google/cloud/bigquery/copy_job.rb +6 -16
- data/lib/google/cloud/bigquery/credentials.rb +5 -12
- data/lib/google/cloud/bigquery/data.rb +10 -16
- data/lib/google/cloud/bigquery/dataset.rb +58 -118
- data/lib/google/cloud/bigquery/dataset/access.rb +5 -13
- data/lib/google/cloud/bigquery/dataset/list.rb +4 -9
- data/lib/google/cloud/bigquery/external.rb +14 -35
- data/lib/google/cloud/bigquery/extract_job.rb +2 -5
- data/lib/google/cloud/bigquery/insert_response.rb +1 -3
- data/lib/google/cloud/bigquery/job.rb +5 -9
- data/lib/google/cloud/bigquery/job/list.rb +3 -7
- data/lib/google/cloud/bigquery/load_job.rb +18 -33
- data/lib/google/cloud/bigquery/model.rb +1 -4
- data/lib/google/cloud/bigquery/model/list.rb +3 -7
- data/lib/google/cloud/bigquery/project.rb +27 -56
- data/lib/google/cloud/bigquery/project/list.rb +3 -7
- data/lib/google/cloud/bigquery/query_job.rb +40 -79
- data/lib/google/cloud/bigquery/schema.rb +3 -8
- data/lib/google/cloud/bigquery/schema/field.rb +6 -11
- data/lib/google/cloud/bigquery/service.rb +26 -58
- data/lib/google/cloud/bigquery/table.rb +58 -112
- data/lib/google/cloud/bigquery/table/async_inserter.rb +10 -18
- data/lib/google/cloud/bigquery/table/list.rb +3 -7
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +36 -42
@@ -205,11 +205,8 @@ module Google
|
|
205
205
|
# end
|
206
206
|
#
|
207
207
|
def query_plan
|
208
|
-
return nil unless @gapi
|
209
|
-
|
210
|
-
Array(@gapi.statistics.query.query_plan).map do |stage|
|
211
|
-
Stage.from_gapi stage
|
212
|
-
end
|
208
|
+
return nil unless @gapi&.statistics&.query&.query_plan
|
209
|
+
Array(@gapi.statistics.query.query_plan).map { |stage| Stage.from_gapi stage }
|
213
210
|
end
|
214
211
|
|
215
212
|
##
|
@@ -261,8 +258,8 @@ module Google
|
|
261
258
|
# query_job.ddl? #=> true
|
262
259
|
#
|
263
260
|
def ddl?
|
264
|
-
|
265
|
-
|
261
|
+
["CREATE_MODEL", "CREATE_TABLE", "CREATE_TABLE_AS_SELECT", "CREATE_VIEW", "\n", "DROP_MODEL", "DROP_TABLE",
|
262
|
+
"DROP_VIEW"].include? statement_type
|
266
263
|
end
|
267
264
|
|
268
265
|
##
|
@@ -285,7 +282,7 @@ module Google
|
|
285
282
|
# query_job.dml? #=> true
|
286
283
|
#
|
287
284
|
def dml?
|
288
|
-
|
285
|
+
["INSERT", "UPDATE", "MERGE", "DELETE"].include? statement_type
|
289
286
|
end
|
290
287
|
|
291
288
|
##
|
@@ -383,9 +380,7 @@ module Google
|
|
383
380
|
def udfs
|
384
381
|
udfs_gapi = @gapi.configuration.query.user_defined_function_resources
|
385
382
|
return nil unless udfs_gapi
|
386
|
-
Array(udfs_gapi).map
|
387
|
-
udf.inline_code || udf.resource_uri
|
388
|
-
end
|
383
|
+
Array(udfs_gapi).map { |udf| udf.inline_code || udf.resource_uri }
|
389
384
|
end
|
390
385
|
|
391
386
|
##
|
@@ -396,9 +391,7 @@ module Google
|
|
396
391
|
#
|
397
392
|
# @!group Attributes
|
398
393
|
def encryption
|
399
|
-
EncryptionConfiguration.from_gapi
|
400
|
-
@gapi.configuration.query.destination_encryption_configuration
|
401
|
-
)
|
394
|
+
EncryptionConfiguration.from_gapi @gapi.configuration.query.destination_encryption_configuration
|
402
395
|
end
|
403
396
|
|
404
397
|
###
|
@@ -535,8 +528,7 @@ module Google
|
|
535
528
|
|
536
529
|
ensure_service!
|
537
530
|
loop do
|
538
|
-
query_results_gapi = service.job_query_results
|
539
|
-
job_id, location: location, max: 0
|
531
|
+
query_results_gapi = service.job_query_results job_id, location: location, max: 0
|
540
532
|
if query_results_gapi.job_complete
|
541
533
|
@destination_schema_gapi = query_results_gapi.schema
|
542
534
|
break
|
@@ -573,9 +565,7 @@ module Google
|
|
573
565
|
#
|
574
566
|
def data token: nil, max: nil, start: nil
|
575
567
|
return nil unless done?
|
576
|
-
if dryrun?
|
577
|
-
return Data.from_gapi_json({ rows: [] }, nil, @gapi, service)
|
578
|
-
end
|
568
|
+
return Data.from_gapi_json({ rows: [] }, nil, @gapi, service) if dryrun?
|
579
569
|
if ddl? || dml?
|
580
570
|
data_hash = { totalRows: nil, rows: [] }
|
581
571
|
return Data.from_gapi_json data_hash, nil, @gapi, service
|
@@ -583,10 +573,7 @@ module Google
|
|
583
573
|
ensure_schema!
|
584
574
|
|
585
575
|
options = { token: token, max: max, start: start }
|
586
|
-
data_hash = service.list_tabledata
|
587
|
-
destination_table_dataset_id,
|
588
|
-
destination_table_table_id,
|
589
|
-
options
|
576
|
+
data_hash = service.list_tabledata destination_table_dataset_id, destination_table_table_id, options
|
590
577
|
Data.from_gapi_json data_hash, destination_table_gapi, @gapi, service
|
591
578
|
end
|
592
579
|
alias query_results data
|
@@ -631,8 +618,7 @@ module Google
|
|
631
618
|
updater.dryrun = options[:dryrun]
|
632
619
|
updater.maximum_bytes_billed = options[:maximum_bytes_billed]
|
633
620
|
updater.labels = options[:labels] if options[:labels]
|
634
|
-
updater.legacy_sql = Convert.resolve_legacy_sql
|
635
|
-
options[:standard_sql], options[:legacy_sql])
|
621
|
+
updater.legacy_sql = Convert.resolve_legacy_sql options[:standard_sql], options[:legacy_sql]
|
636
622
|
updater.external = options[:external] if options[:external]
|
637
623
|
updater.priority = options[:priority]
|
638
624
|
updater.cache = options[:cache]
|
@@ -731,8 +717,7 @@ module Google
|
|
731
717
|
#
|
732
718
|
# @!group Attributes
|
733
719
|
def dataset= value
|
734
|
-
@gapi.configuration.query.default_dataset =
|
735
|
-
@service.dataset_ref_from value
|
720
|
+
@gapi.configuration.query.default_dataset = @service.dataset_ref_from value
|
736
721
|
end
|
737
722
|
|
738
723
|
##
|
@@ -752,17 +737,13 @@ module Google
|
|
752
737
|
when Array then
|
753
738
|
@gapi.configuration.query.use_legacy_sql = false
|
754
739
|
@gapi.configuration.query.parameter_mode = "POSITIONAL"
|
755
|
-
@gapi.configuration.query.query_parameters = params.map
|
756
|
-
Convert.to_query_param param
|
757
|
-
end
|
740
|
+
@gapi.configuration.query.query_parameters = params.map { |param| Convert.to_query_param param }
|
758
741
|
when Hash then
|
759
742
|
@gapi.configuration.query.use_legacy_sql = false
|
760
743
|
@gapi.configuration.query.parameter_mode = "NAMED"
|
761
744
|
@gapi.configuration.query.query_parameters =
|
762
745
|
params.map do |name, param|
|
763
|
-
Convert.to_query_param(param).tap
|
764
|
-
named_param.name = String name
|
765
|
-
end
|
746
|
+
Convert.to_query_param(param).tap { |named_param| named_param.name = String name }
|
766
747
|
end
|
767
748
|
else
|
768
749
|
raise "Query parameters must be an Array or a Hash."
|
@@ -783,8 +764,7 @@ module Google
|
|
783
764
|
#
|
784
765
|
# @!group Attributes
|
785
766
|
def create= value
|
786
|
-
@gapi.configuration.query.create_disposition =
|
787
|
-
Convert.create_disposition value
|
767
|
+
@gapi.configuration.query.create_disposition = Convert.create_disposition value
|
788
768
|
end
|
789
769
|
|
790
770
|
##
|
@@ -802,8 +782,7 @@ module Google
|
|
802
782
|
#
|
803
783
|
# @!group Attributes
|
804
784
|
def write= value
|
805
|
-
@gapi.configuration.query.write_disposition =
|
806
|
-
Convert.write_disposition value
|
785
|
+
@gapi.configuration.query.write_disposition = Convert.write_disposition value
|
807
786
|
end
|
808
787
|
|
809
788
|
##
|
@@ -905,9 +884,7 @@ module Google
|
|
905
884
|
# @!group Attributes
|
906
885
|
#
|
907
886
|
def external= value
|
908
|
-
external_table_pairs = value.map
|
909
|
-
[String(name), obj.to_gapi]
|
910
|
-
end
|
887
|
+
external_table_pairs = value.map { |name, obj| [String(name), obj.to_gapi] }
|
911
888
|
external_table_hash = Hash[external_table_pairs]
|
912
889
|
@gapi.configuration.query.table_definitions = external_table_hash
|
913
890
|
end
|
@@ -925,8 +902,7 @@ module Google
|
|
925
902
|
#
|
926
903
|
# @!group Attributes
|
927
904
|
def udfs= value
|
928
|
-
@gapi.configuration.query.user_defined_function_resources =
|
929
|
-
udfs_gapi_from value
|
905
|
+
@gapi.configuration.query.user_defined_function_resources = udfs_gapi_from value
|
930
906
|
end
|
931
907
|
|
932
908
|
##
|
@@ -950,9 +926,7 @@ module Google
|
|
950
926
|
#
|
951
927
|
# @!group Attributes
|
952
928
|
def encryption= val
|
953
|
-
@gapi.configuration.query.update!
|
954
|
-
destination_encryption_configuration: val.to_gapi
|
955
|
-
)
|
929
|
+
@gapi.configuration.query.update! destination_encryption_configuration: val.to_gapi
|
956
930
|
end
|
957
931
|
|
958
932
|
##
|
@@ -989,8 +963,7 @@ module Google
|
|
989
963
|
# @!group Attributes
|
990
964
|
#
|
991
965
|
def time_partitioning_type= type
|
992
|
-
@gapi.configuration.query.time_partitioning ||=
|
993
|
-
Google::Apis::BigqueryV2::TimePartitioning.new
|
966
|
+
@gapi.configuration.query.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
|
994
967
|
@gapi.configuration.query.time_partitioning.update! type: type
|
995
968
|
end
|
996
969
|
|
@@ -1036,8 +1009,7 @@ module Google
|
|
1036
1009
|
# @!group Attributes
|
1037
1010
|
#
|
1038
1011
|
def time_partitioning_field= field
|
1039
|
-
@gapi.configuration.query.time_partitioning ||=
|
1040
|
-
Google::Apis::BigqueryV2::TimePartitioning.new
|
1012
|
+
@gapi.configuration.query.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
|
1041
1013
|
@gapi.configuration.query.time_partitioning.update! field: field
|
1042
1014
|
end
|
1043
1015
|
|
@@ -1076,10 +1048,8 @@ module Google
|
|
1076
1048
|
# @!group Attributes
|
1077
1049
|
#
|
1078
1050
|
def time_partitioning_expiration= expiration
|
1079
|
-
@gapi.configuration.query.time_partitioning ||=
|
1080
|
-
|
1081
|
-
@gapi.configuration.query.time_partitioning.update! \
|
1082
|
-
expiration_ms: expiration * 1000
|
1051
|
+
@gapi.configuration.query.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
|
1052
|
+
@gapi.configuration.query.time_partitioning.update! expiration_ms: expiration * 1000
|
1083
1053
|
end
|
1084
1054
|
|
1085
1055
|
##
|
@@ -1094,10 +1064,8 @@ module Google
|
|
1094
1064
|
# @!group Attributes
|
1095
1065
|
#
|
1096
1066
|
def time_partitioning_require_filter= val
|
1097
|
-
@gapi.configuration.query.time_partitioning ||=
|
1098
|
-
|
1099
|
-
@gapi.configuration.query.time_partitioning.update! \
|
1100
|
-
require_partition_filter: val
|
1067
|
+
@gapi.configuration.query.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
|
1068
|
+
@gapi.configuration.query.time_partitioning.update! require_partition_filter: val
|
1101
1069
|
end
|
1102
1070
|
|
1103
1071
|
##
|
@@ -1143,8 +1111,7 @@ module Google
|
|
1143
1111
|
# @!group Attributes
|
1144
1112
|
#
|
1145
1113
|
def clustering_fields= fields
|
1146
|
-
@gapi.configuration.query.clustering ||=
|
1147
|
-
Google::Apis::BigqueryV2::Clustering.new
|
1114
|
+
@gapi.configuration.query.clustering ||= Google::Apis::BigqueryV2::Clustering.new
|
1148
1115
|
@gapi.configuration.query.clustering.fields = fields
|
1149
1116
|
end
|
1150
1117
|
|
@@ -1170,14 +1137,12 @@ module Google
|
|
1170
1137
|
end
|
1171
1138
|
|
1172
1139
|
def priority_value str
|
1173
|
-
{ "batch"
|
1174
|
-
"interactive" => "INTERACTIVE" }[str.to_s.downcase]
|
1140
|
+
{ "batch" => "BATCH", "interactive" => "INTERACTIVE" }[str.to_s.downcase]
|
1175
1141
|
end
|
1176
1142
|
|
1177
1143
|
def udfs_gapi_from array_or_str
|
1178
1144
|
Array(array_or_str).map do |uri_or_code|
|
1179
|
-
resource =
|
1180
|
-
Google::Apis::BigqueryV2::UserDefinedFunctionResource.new
|
1145
|
+
resource = Google::Apis::BigqueryV2::UserDefinedFunctionResource.new
|
1181
1146
|
if uri_or_code.start_with? "gs://"
|
1182
1147
|
resource.resource_uri = uri_or_code
|
1183
1148
|
else
|
@@ -1237,17 +1202,15 @@ module Google
|
|
1237
1202
|
# end
|
1238
1203
|
#
|
1239
1204
|
class Stage
|
1240
|
-
attr_reader :compute_ratio_avg, :compute_ratio_max, :id, :name,
|
1241
|
-
:
|
1242
|
-
:
|
1243
|
-
:wait_ratio_max, :write_ratio_avg, :write_ratio_max
|
1205
|
+
attr_reader :compute_ratio_avg, :compute_ratio_max, :id, :name, :read_ratio_avg, :read_ratio_max,
|
1206
|
+
:records_read, :records_written, :status, :steps, :wait_ratio_avg, :wait_ratio_max,
|
1207
|
+
:write_ratio_avg, :write_ratio_max
|
1244
1208
|
|
1245
1209
|
##
|
1246
1210
|
# @private Creates a new Stage instance.
|
1247
|
-
def initialize compute_ratio_avg, compute_ratio_max, id, name,
|
1248
|
-
|
1249
|
-
|
1250
|
-
wait_ratio_max, write_ratio_avg, write_ratio_max
|
1211
|
+
def initialize compute_ratio_avg, compute_ratio_max, id, name, read_ratio_avg, read_ratio_max, records_read,
|
1212
|
+
records_written, status, steps, wait_ratio_avg, wait_ratio_max, write_ratio_avg,
|
1213
|
+
write_ratio_max
|
1251
1214
|
@compute_ratio_avg = compute_ratio_avg
|
1252
1215
|
@compute_ratio_max = compute_ratio_max
|
1253
1216
|
@id = id
|
@@ -1268,11 +1231,9 @@ module Google
|
|
1268
1231
|
# @private New Stage from a statistics.query.queryPlan element.
|
1269
1232
|
def self.from_gapi gapi
|
1270
1233
|
steps = Array(gapi.steps).map { |g| Step.from_gapi g }
|
1271
|
-
new gapi.compute_ratio_avg, gapi.compute_ratio_max, gapi.id,
|
1272
|
-
gapi.
|
1273
|
-
gapi.
|
1274
|
-
gapi.wait_ratio_avg, gapi.wait_ratio_max, gapi.write_ratio_avg,
|
1275
|
-
gapi.write_ratio_max
|
1234
|
+
new gapi.compute_ratio_avg, gapi.compute_ratio_max, gapi.id, gapi.name, gapi.read_ratio_avg,
|
1235
|
+
gapi.read_ratio_max, gapi.records_read, gapi.records_written, gapi.status, steps, gapi.wait_ratio_avg,
|
1236
|
+
gapi.wait_ratio_max, gapi.write_ratio_avg, gapi.write_ratio_max
|
1276
1237
|
end
|
1277
1238
|
end
|
1278
1239
|
|
@@ -1327,8 +1288,7 @@ module Google
|
|
1327
1288
|
def ensure_schema!
|
1328
1289
|
return unless destination_schema.nil?
|
1329
1290
|
|
1330
|
-
query_results_gapi = service.job_query_results
|
1331
|
-
job_id, location: location, max: 0
|
1291
|
+
query_results_gapi = service.job_query_results job_id, location: location, max: 0
|
1332
1292
|
# raise "unable to retrieve schema" if query_results_gapi.schema.nil?
|
1333
1293
|
@destination_schema_gapi = query_results_gapi.schema
|
1334
1294
|
end
|
@@ -1346,9 +1306,10 @@ module Google
|
|
1346
1306
|
end
|
1347
1307
|
|
1348
1308
|
def destination_table_gapi
|
1349
|
-
Google::Apis::BigqueryV2::Table.new
|
1309
|
+
Google::Apis::BigqueryV2::Table.new(
|
1350
1310
|
table_reference: @gapi.configuration.query.destination_table,
|
1351
1311
|
schema: destination_schema
|
1312
|
+
)
|
1352
1313
|
end
|
1353
1314
|
end
|
1354
1315
|
end
|
@@ -461,8 +461,7 @@ module Google
|
|
461
461
|
# TODO: do we need to raise if no block was given?
|
462
462
|
raise ArgumentError, "a block is required" unless block_given?
|
463
463
|
|
464
|
-
nested_field = add_field name, :record, description: description,
|
465
|
-
mode: mode
|
464
|
+
nested_field = add_field name, :record, description: description, mode: mode
|
466
465
|
yield nested_field
|
467
466
|
nested_field
|
468
467
|
end
|
@@ -528,18 +527,14 @@ module Google
|
|
528
527
|
|
529
528
|
def verify_type type
|
530
529
|
type = type.to_s.upcase
|
531
|
-
unless Field::TYPES.include? type
|
532
|
-
raise ArgumentError, "Type '#{type}' not found"
|
533
|
-
end
|
530
|
+
raise ArgumentError, "Type '#{type}' not found" unless Field::TYPES.include? type
|
534
531
|
type
|
535
532
|
end
|
536
533
|
|
537
534
|
def verify_mode mode
|
538
535
|
mode = :nullable if mode.nil?
|
539
536
|
mode = mode.to_s.upcase
|
540
|
-
unless Field::MODES.include? mode
|
541
|
-
raise ArgumentError "Unable to determine mode for '#{mode}'"
|
542
|
-
end
|
537
|
+
raise ArgumentError "Unable to determine mode for '#{mode}'" unless Field::MODES.include? mode
|
543
538
|
mode
|
544
539
|
end
|
545
540
|
end
|
@@ -37,11 +37,11 @@ module Google
|
|
37
37
|
#
|
38
38
|
class Field
|
39
39
|
# @private
|
40
|
-
MODES =
|
40
|
+
MODES = ["NULLABLE", "REQUIRED", "REPEATED"].freeze
|
41
41
|
|
42
42
|
# @private
|
43
|
-
TYPES =
|
44
|
-
|
43
|
+
TYPES = ["STRING", "INTEGER", "INT64", "FLOAT", "FLOAT64", "NUMERIC", "BOOLEAN", "BOOL", "BYTES", "TIMESTAMP",
|
44
|
+
"TIME", "DATETIME", "DATE", "RECORD", "STRUCT"].freeze
|
45
45
|
|
46
46
|
##
|
47
47
|
# The name of the field.
|
@@ -548,8 +548,7 @@ module Google
|
|
548
548
|
# TODO: do we need to raise if no block was given?
|
549
549
|
raise ArgumentError, "a block is required" unless block_given?
|
550
550
|
|
551
|
-
nested_field = add_field name, :record, description: description,
|
552
|
-
mode: mode
|
551
|
+
nested_field = add_field name, :record, description: description, mode: mode
|
553
552
|
yield nested_field
|
554
553
|
nested_field
|
555
554
|
end
|
@@ -621,18 +620,14 @@ module Google
|
|
621
620
|
|
622
621
|
def verify_type type
|
623
622
|
type = type.to_s.upcase
|
624
|
-
unless TYPES.include? type
|
625
|
-
raise ArgumentError, "Type '#{type}' not found"
|
626
|
-
end
|
623
|
+
raise ArgumentError, "Type '#{type}' not found" unless TYPES.include? type
|
627
624
|
type
|
628
625
|
end
|
629
626
|
|
630
627
|
def verify_mode mode
|
631
628
|
mode = :nullable if mode.nil?
|
632
629
|
mode = mode.to_s.upcase
|
633
|
-
unless MODES.include? mode
|
634
|
-
raise ArgumentError "Unable to determine mode for '#{mode}'"
|
635
|
-
end
|
630
|
+
raise ArgumentError "Unable to determine mode for '#{mode}'" unless MODES.include? mode
|
636
631
|
mode
|
637
632
|
end
|
638
633
|
end
|
@@ -43,8 +43,7 @@ module Google
|
|
43
43
|
|
44
44
|
##
|
45
45
|
# Creates a new Service instance.
|
46
|
-
def initialize project, credentials,
|
47
|
-
retries: nil, timeout: nil, host: nil
|
46
|
+
def initialize project, credentials, retries: nil, timeout: nil, host: nil
|
48
47
|
@project = project
|
49
48
|
@credentials = credentials
|
50
49
|
@retries = retries
|
@@ -57,8 +56,7 @@ module Google
|
|
57
56
|
@service ||= begin
|
58
57
|
service = API::BigqueryService.new
|
59
58
|
service.client_options.application_name = "gcloud-ruby"
|
60
|
-
service.client_options.application_version =
|
61
|
-
Google::Cloud::Bigquery::VERSION
|
59
|
+
service.client_options.application_version = Google::Cloud::Bigquery::VERSION
|
62
60
|
service.client_options.open_timeout_sec = timeout
|
63
61
|
service.client_options.read_timeout_sec = timeout
|
64
62
|
service.client_options.send_timeout_sec = timeout
|
@@ -116,8 +114,7 @@ module Google
|
|
116
114
|
patch_with_backoff = true
|
117
115
|
end
|
118
116
|
execute backoff: patch_with_backoff do
|
119
|
-
service.patch_dataset @project, dataset_id, patched_dataset_gapi,
|
120
|
-
options: options
|
117
|
+
service.patch_dataset @project, dataset_id, patched_dataset_gapi, options: options
|
121
118
|
end
|
122
119
|
end
|
123
120
|
|
@@ -139,9 +136,7 @@ module Google
|
|
139
136
|
def list_tables dataset_id, options = {}
|
140
137
|
# The list operation is considered idempotent
|
141
138
|
execute backoff: true do
|
142
|
-
service.list_tables @project, dataset_id,
|
143
|
-
max_results: options[:max],
|
144
|
-
page_token: options[:token]
|
139
|
+
service.list_tables @project, dataset_id, max_results: options[:max], page_token: options[:token]
|
145
140
|
end
|
146
141
|
end
|
147
142
|
|
@@ -182,8 +177,7 @@ module Google
|
|
182
177
|
patch_with_backoff = true
|
183
178
|
end
|
184
179
|
execute backoff: patch_with_backoff do
|
185
|
-
service.patch_table @project, dataset_id, table_id,
|
186
|
-
patched_table_gapi, options: options
|
180
|
+
service.patch_table @project, dataset_id, table_id, patched_table_gapi, options: options
|
187
181
|
end
|
188
182
|
end
|
189
183
|
|
@@ -214,9 +208,7 @@ module Google
|
|
214
208
|
insert_tabledata_json_rows dataset_id, table_id, json_rows, options
|
215
209
|
end
|
216
210
|
|
217
|
-
def insert_tabledata_json_rows dataset_id, table_id, json_rows,
|
218
|
-
options = {}
|
219
|
-
|
211
|
+
def insert_tabledata_json_rows dataset_id, table_id, json_rows, options = {}
|
220
212
|
rows_and_ids = Array(json_rows).zip Array(options[:insert_ids])
|
221
213
|
insert_rows = rows_and_ids.map do |json_row, insert_id|
|
222
214
|
insert_id ||= SecureRandom.uuid
|
@@ -248,10 +240,7 @@ module Google
|
|
248
240
|
options = { skip_deserialization: true }
|
249
241
|
# The list operation is considered idempotent
|
250
242
|
execute backoff: true do
|
251
|
-
json_txt = service.list_models @project, dataset_id,
|
252
|
-
max_results: max,
|
253
|
-
page_token: token,
|
254
|
-
options: options
|
243
|
+
json_txt = service.list_models @project, dataset_id, max_results: max, page_token: token, options: options
|
255
244
|
JSON.parse json_txt, symbolize_names: true
|
256
245
|
end
|
257
246
|
end
|
@@ -263,8 +252,7 @@ module Google
|
|
263
252
|
def get_model dataset_id, model_id
|
264
253
|
# The get operation is considered idempotent
|
265
254
|
execute backoff: true do
|
266
|
-
json_txt = service.get_model @project, dataset_id, model_id,
|
267
|
-
options: { skip_deserialization: true }
|
255
|
+
json_txt = service.get_model @project, dataset_id, model_id, options: { skip_deserialization: true }
|
268
256
|
JSON.parse json_txt, symbolize_names: true
|
269
257
|
end
|
270
258
|
end
|
@@ -281,9 +269,7 @@ module Google
|
|
281
269
|
patch_with_backoff = true
|
282
270
|
end
|
283
271
|
execute backoff: patch_with_backoff do
|
284
|
-
json_txt = service.patch_model @project, dataset_id, model_id,
|
285
|
-
patched_model_gapi,
|
286
|
-
options: options
|
272
|
+
json_txt = service.patch_model @project, dataset_id, model_id, patched_model_gapi, options: options
|
287
273
|
JSON.parse json_txt, symbolize_names: true
|
288
274
|
end
|
289
275
|
end
|
@@ -303,12 +289,9 @@ module Google
|
|
303
289
|
min_creation_time = Convert.time_to_millis options[:min_created_at]
|
304
290
|
max_creation_time = Convert.time_to_millis options[:max_created_at]
|
305
291
|
execute backoff: true do
|
306
|
-
service.list_jobs
|
307
|
-
|
308
|
-
|
309
|
-
state_filter: options[:filter],
|
310
|
-
min_creation_time: min_creation_time,
|
311
|
-
max_creation_time: max_creation_time
|
292
|
+
service.list_jobs @project, all_users: options[:all], max_results: options[:max],
|
293
|
+
page_token: options[:token], projection: "full", state_filter: options[:filter],
|
294
|
+
min_creation_time: min_creation_time, max_creation_time: max_creation_time
|
312
295
|
end
|
313
296
|
end
|
314
297
|
|
@@ -331,10 +314,7 @@ module Google
|
|
331
314
|
end
|
332
315
|
|
333
316
|
def insert_job config, location: nil
|
334
|
-
job_object = API::Job.new(
|
335
|
-
job_reference: job_ref_from(nil, nil, location: location),
|
336
|
-
configuration: config
|
337
|
-
)
|
317
|
+
job_object = API::Job.new job_reference: job_ref_from(nil, nil, location: location), configuration: config
|
338
318
|
# Jobs have generated id, so this operation is considered idempotent
|
339
319
|
execute backoff: true do
|
340
320
|
service.insert_job @project, job_object
|
@@ -352,13 +332,12 @@ module Google
|
|
352
332
|
def job_query_results job_id, options = {}
|
353
333
|
# The get operation is considered idempotent
|
354
334
|
execute backoff: true do
|
355
|
-
service.get_job_query_results
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
timeout_ms: options.delete(:timeout)
|
335
|
+
service.get_job_query_results @project, job_id,
|
336
|
+
location: options.delete(:location),
|
337
|
+
max_results: options.delete(:max),
|
338
|
+
page_token: options.delete(:token),
|
339
|
+
start_index: options.delete(:start),
|
340
|
+
timeout_ms: options.delete(:timeout)
|
362
341
|
end
|
363
342
|
end
|
364
343
|
|
@@ -382,10 +361,7 @@ module Google
|
|
382
361
|
|
383
362
|
def load_table_file file, load_job_gapi
|
384
363
|
execute backoff: true do
|
385
|
-
service.insert_job
|
386
|
-
@project,
|
387
|
-
load_job_gapi,
|
388
|
-
upload_source: file, content_type: mime_type_for(file)
|
364
|
+
service.insert_job @project, load_job_gapi, upload_source: file, content_type: mime_type_for(file)
|
389
365
|
end
|
390
366
|
end
|
391
367
|
|
@@ -409,9 +385,7 @@ module Google
|
|
409
385
|
def self.table_ref_from_s str, default_ref: {}
|
410
386
|
str = str.to_s
|
411
387
|
m = /\A(((?<prj>\S*)(:|\.))?(?<dts>\S*)\.)?(?<tbl>\S*)\z/.match str
|
412
|
-
unless m
|
413
|
-
raise ArgumentError, "unable to identify table from #{str.inspect}"
|
414
|
-
end
|
388
|
+
raise ArgumentError, "unable to identify table from #{str.inspect}" unless m
|
415
389
|
str_table_ref_hash = {
|
416
390
|
project_id: m["prj"],
|
417
391
|
dataset_id: m["dts"],
|
@@ -424,10 +398,8 @@ module Google
|
|
424
398
|
end
|
425
399
|
|
426
400
|
def self.validate_table_ref table_ref
|
427
|
-
|
428
|
-
if table_ref.send(f).nil?
|
429
|
-
raise ArgumentError, "TableReference is missing #{f}"
|
430
|
-
end
|
401
|
+
[:project_id, :dataset_id, :table_id].each do |f|
|
402
|
+
raise ArgumentError, "TableReference is missing #{f}" if table_ref.send(f).nil?
|
431
403
|
end
|
432
404
|
end
|
433
405
|
|
@@ -435,8 +407,7 @@ module Google
|
|
435
407
|
# Lists all projects to which you have been granted any project role.
|
436
408
|
def list_projects options = {}
|
437
409
|
execute backoff: true do
|
438
|
-
service.list_projects max_results: options[:max],
|
439
|
-
page_token: options[:token]
|
410
|
+
service.list_projects max_results: options[:max], page_token: options[:token]
|
440
411
|
end
|
441
412
|
end
|
442
413
|
|
@@ -446,10 +417,7 @@ module Google
|
|
446
417
|
def job_ref_from job_id, prefix, location: nil
|
447
418
|
prefix ||= "job_"
|
448
419
|
job_id ||= "#{prefix}#{generate_id}"
|
449
|
-
job_ref = API::JobReference.new
|
450
|
-
project_id: @project,
|
451
|
-
job_id: job_id
|
452
|
-
)
|
420
|
+
job_ref = API::JobReference.new project_id: @project, job_id: job_id
|
453
421
|
# BigQuery does not allow nil location, but missing is ok.
|
454
422
|
job_ref.location = location if location
|
455
423
|
job_ref
|
@@ -507,7 +475,7 @@ module Google
|
|
507
475
|
attr_accessor :backoff
|
508
476
|
end
|
509
477
|
self.retries = 5
|
510
|
-
self.reasons =
|
478
|
+
self.reasons = ["rateLimitExceeded", "backendError"]
|
511
479
|
self.backoff = lambda do |retries|
|
512
480
|
# Max delay is 32 seconds
|
513
481
|
# See "Back-off Requirements" here:
|