google-cloud-bigquery 1.16.0 → 1.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. checksums.yaml +4 -4
  2. data/AUTHENTICATION.md +7 -27
  3. data/CHANGELOG.md +8 -0
  4. data/CONTRIBUTING.md +1 -1
  5. data/OVERVIEW.md +5 -5
  6. data/lib/google-cloud-bigquery.rb +5 -11
  7. data/lib/google/cloud/bigquery.rb +3 -5
  8. data/lib/google/cloud/bigquery/convert.rb +23 -46
  9. data/lib/google/cloud/bigquery/copy_job.rb +6 -16
  10. data/lib/google/cloud/bigquery/credentials.rb +5 -12
  11. data/lib/google/cloud/bigquery/data.rb +10 -16
  12. data/lib/google/cloud/bigquery/dataset.rb +58 -118
  13. data/lib/google/cloud/bigquery/dataset/access.rb +5 -13
  14. data/lib/google/cloud/bigquery/dataset/list.rb +4 -9
  15. data/lib/google/cloud/bigquery/external.rb +14 -35
  16. data/lib/google/cloud/bigquery/extract_job.rb +2 -5
  17. data/lib/google/cloud/bigquery/insert_response.rb +1 -3
  18. data/lib/google/cloud/bigquery/job.rb +5 -9
  19. data/lib/google/cloud/bigquery/job/list.rb +3 -7
  20. data/lib/google/cloud/bigquery/load_job.rb +18 -33
  21. data/lib/google/cloud/bigquery/model.rb +1 -4
  22. data/lib/google/cloud/bigquery/model/list.rb +3 -7
  23. data/lib/google/cloud/bigquery/project.rb +27 -56
  24. data/lib/google/cloud/bigquery/project/list.rb +3 -7
  25. data/lib/google/cloud/bigquery/query_job.rb +40 -79
  26. data/lib/google/cloud/bigquery/schema.rb +3 -8
  27. data/lib/google/cloud/bigquery/schema/field.rb +6 -11
  28. data/lib/google/cloud/bigquery/service.rb +26 -58
  29. data/lib/google/cloud/bigquery/table.rb +58 -112
  30. data/lib/google/cloud/bigquery/table/async_inserter.rb +10 -18
  31. data/lib/google/cloud/bigquery/table/list.rb +3 -7
  32. data/lib/google/cloud/bigquery/version.rb +1 -1
  33. metadata +36 -42
@@ -57,9 +57,7 @@ module Google
57
57
  def source
58
58
  table = @gapi.configuration.extract.source_table
59
59
  return nil unless table
60
- retrieve_table table.project_id,
61
- table.dataset_id,
62
- table.table_id
60
+ retrieve_table table.project_id, table.dataset_id, table.table_id
63
61
  end
64
62
 
65
63
  ##
@@ -291,8 +289,7 @@ module Google
291
289
  # @!group Attributes
292
290
  #
293
291
  def format= new_format
294
- @gapi.configuration.extract.update! \
295
- destination_format: Convert.source_format(new_format)
292
+ @gapi.configuration.extract.update! destination_format: Convert.source_format(new_format)
296
293
  end
297
294
 
298
295
  ##
@@ -99,9 +99,7 @@ module Google
99
99
  # data.
100
100
  #
101
101
  def error_rows
102
- Array(@gapi.insert_errors).map do |ie|
103
- @rows[ie.index]
104
- end
102
+ Array(@gapi.insert_errors).map { |ie| @rows[ie.index] }
105
103
  end
106
104
 
107
105
  ##
@@ -440,15 +440,11 @@ module Google
440
440
  end
441
441
 
442
442
  def status_code_for_reason reason
443
- codes = { "accessDenied" => 403, "backendError" => 500,
444
- "billingNotEnabled" => 403,
445
- "billingTierLimitExceeded" => 400, "blocked" => 403,
446
- "duplicate" => 409, "internalError" => 500,
447
- "invalid" => 400, "invalidQuery" => 400, "notFound" => 404,
448
- "notImplemented" => 501, "quotaExceeded" => 403,
449
- "rateLimitExceeded" => 403, "resourceInUse" => 400,
450
- "resourcesExceeded" => 400, "responseTooLarge" => 403,
451
- "tableUnavailable" => 400 }
443
+ codes = { "accessDenied" => 403, "backendError" => 500, "billingNotEnabled" => 403,
444
+ "billingTierLimitExceeded" => 400, "blocked" => 403, "duplicate" => 409, "internalError" => 500,
445
+ "invalid" => 400, "invalidQuery" => 400, "notFound" => 404, "notImplemented" => 501,
446
+ "quotaExceeded" => 403, "rateLimitExceeded" => 403, "resourceInUse" => 400,
447
+ "resourcesExceeded" => 400, "responseTooLarge" => 403, "tableUnavailable" => 400 }
452
448
  codes[reason] || 0
453
449
  end
454
450
  end
@@ -123,15 +123,13 @@ module Google
123
123
  #
124
124
  def all request_limit: nil
125
125
  request_limit = request_limit.to_i if request_limit
126
- unless block_given?
127
- return enum_for :all, request_limit: request_limit
128
- end
126
+ return enum_for :all, request_limit: request_limit unless block_given?
129
127
  results = self
130
128
  loop do
131
129
  results.each { |r| yield r }
132
130
  if request_limit
133
131
  request_limit -= 1
134
- break if request_limit < 0
132
+ break if request_limit.negative?
135
133
  end
136
134
  break unless results.next?
137
135
  results = results.next
@@ -142,9 +140,7 @@ module Google
142
140
  # @private New Job::List from a Google API Client
143
141
  # Google::Apis::BigqueryV2::JobList object.
144
142
  def self.from_gapi gapi_list, service, options = {}
145
- jobs = List.new(Array(gapi_list.jobs).map do |gapi_object|
146
- Job.from_gapi gapi_object, service
147
- end)
143
+ jobs = List.new(Array(gapi_list.jobs).map { |gapi_object| Job.from_gapi gapi_object, service })
148
144
  jobs.instance_variable_set :@token, gapi_list.next_page_token
149
145
  jobs.instance_variable_set :@etag, gapi_list.etag
150
146
  jobs.instance_variable_set :@service, service
@@ -67,9 +67,7 @@ module Google
67
67
  def destination
68
68
  table = @gapi.configuration.load.destination_table
69
69
  return nil unless table
70
- retrieve_table table.project_id,
71
- table.dataset_id,
72
- table.table_id
70
+ retrieve_table table.project_id, table.dataset_id, table.table_id
73
71
  end
74
72
 
75
73
  ##
@@ -402,9 +400,10 @@ module Google
402
400
  # @!group Attributes
403
401
  #
404
402
  def time_partitioning_expiration
405
- @gapi.configuration.load.time_partitioning.expiration_ms / 1_000 if
406
- time_partitioning? &&
407
- !@gapi.configuration.load.time_partitioning.expiration_ms.nil?
403
+ return nil unless time_partitioning?
404
+ return nil if @gapi.configuration.load.time_partitioning.expiration_ms.nil?
405
+
406
+ @gapi.configuration.load.time_partitioning.expiration_ms / 1_000
408
407
  end
409
408
 
410
409
  ###
@@ -934,8 +933,7 @@ module Google
934
933
  # @!group Attributes
935
934
  #
936
935
  def format= new_format
937
- @gapi.configuration.load.update! \
938
- source_format: Convert.source_format(new_format)
936
+ @gapi.configuration.load.update! source_format: Convert.source_format(new_format)
939
937
  end
940
938
 
941
939
  ##
@@ -955,8 +953,7 @@ module Google
955
953
  # @!group Attributes
956
954
  #
957
955
  def create= new_create
958
- @gapi.configuration.load.update! \
959
- create_disposition: Convert.create_disposition(new_create)
956
+ @gapi.configuration.load.update! create_disposition: Convert.create_disposition(new_create)
960
957
  end
961
958
 
962
959
  ##
@@ -977,8 +974,7 @@ module Google
977
974
  # @!group Attributes
978
975
  #
979
976
  def write= new_write
980
- @gapi.configuration.load.update! \
981
- write_disposition: Convert.write_disposition(new_write)
977
+ @gapi.configuration.load.update! write_disposition: Convert.write_disposition(new_write)
982
978
  end
983
979
 
984
980
  ##
@@ -999,8 +995,7 @@ module Google
999
995
  if new_fields.nil?
1000
996
  @gapi.configuration.load.update! projection_fields: nil
1001
997
  else
1002
- @gapi.configuration.load.update! \
1003
- projection_fields: Array(new_fields)
998
+ @gapi.configuration.load.update! projection_fields: Array(new_fields)
1004
999
  end
1005
1000
  end
1006
1001
 
@@ -1199,8 +1194,7 @@ module Google
1199
1194
  if new_options.nil?
1200
1195
  @gapi.configuration.load.update! schema_update_options: nil
1201
1196
  else
1202
- @gapi.configuration.load.update! \
1203
- schema_update_options: Array(new_options)
1197
+ @gapi.configuration.load.update! schema_update_options: Array(new_options)
1204
1198
  end
1205
1199
  end
1206
1200
 
@@ -1238,9 +1232,7 @@ module Google
1238
1232
  #
1239
1233
  # @!group Attributes
1240
1234
  def encryption= val
1241
- @gapi.configuration.load.update!(
1242
- destination_encryption_configuration: val.to_gapi
1243
- )
1235
+ @gapi.configuration.load.update! destination_encryption_configuration: val.to_gapi
1244
1236
  end
1245
1237
 
1246
1238
  ##
@@ -1288,8 +1280,7 @@ module Google
1288
1280
  # @!group Attributes
1289
1281
  #
1290
1282
  def time_partitioning_type= type
1291
- @gapi.configuration.load.time_partitioning ||= \
1292
- Google::Apis::BigqueryV2::TimePartitioning.new
1283
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
1293
1284
  @gapi.configuration.load.time_partitioning.update! type: type
1294
1285
  end
1295
1286
 
@@ -1332,8 +1323,7 @@ module Google
1332
1323
  # @!group Attributes
1333
1324
  #
1334
1325
  def time_partitioning_field= field
1335
- @gapi.configuration.load.time_partitioning ||= \
1336
- Google::Apis::BigqueryV2::TimePartitioning.new
1326
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
1337
1327
  @gapi.configuration.load.time_partitioning.update! field: field
1338
1328
  end
1339
1329
 
@@ -1366,10 +1356,8 @@ module Google
1366
1356
  # @!group Attributes
1367
1357
  #
1368
1358
  def time_partitioning_expiration= expiration
1369
- @gapi.configuration.load.time_partitioning ||= \
1370
- Google::Apis::BigqueryV2::TimePartitioning.new
1371
- @gapi.configuration.load.time_partitioning.update! \
1372
- expiration_ms: expiration * 1000
1359
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
1360
+ @gapi.configuration.load.time_partitioning.update! expiration_ms: expiration * 1000
1373
1361
  end
1374
1362
 
1375
1363
  ##
@@ -1384,10 +1372,8 @@ module Google
1384
1372
  # @!group Attributes
1385
1373
  #
1386
1374
  def time_partitioning_require_filter= val
1387
- @gapi.configuration.load.time_partitioning ||= \
1388
- Google::Apis::BigqueryV2::TimePartitioning.new
1389
- @gapi.configuration.load.time_partitioning.update! \
1390
- require_partition_filter: val
1375
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
1376
+ @gapi.configuration.load.time_partitioning.update! require_partition_filter: val
1391
1377
  end
1392
1378
 
1393
1379
  ##
@@ -1436,8 +1422,7 @@ module Google
1436
1422
  # @!group Attributes
1437
1423
  #
1438
1424
  def clustering_fields= fields
1439
- @gapi.configuration.load.clustering ||= \
1440
- Google::Apis::BigqueryV2::Clustering.new
1425
+ @gapi.configuration.load.clustering ||= Google::Apis::BigqueryV2::Clustering.new
1441
1426
  @gapi.configuration.load.clustering.fields = fields
1442
1427
  end
1443
1428
 
@@ -397,10 +397,7 @@ module Google
397
397
  # We have to create a gapic object from the hash because that is what
398
398
  # EncryptionConfiguration is expecing.
399
399
  json_cmek = @gapi_json[:encryptionConfiguration].to_json
400
- gapi_cmek = \
401
- Google::Apis::BigqueryV2::EncryptionConfiguration.from_json(
402
- json_cmek
403
- )
400
+ gapi_cmek = Google::Apis::BigqueryV2::EncryptionConfiguration.from_json json_cmek
404
401
  EncryptionConfiguration.from_gapi(gapi_cmek).freeze
405
402
  end
406
403
 
@@ -126,15 +126,13 @@ module Google
126
126
  #
127
127
  def all request_limit: nil
128
128
  request_limit = request_limit.to_i if request_limit
129
- unless block_given?
130
- return enum_for :all, request_limit: request_limit
131
- end
129
+ return enum_for :all, request_limit: request_limit unless block_given?
132
130
  results = self
133
131
  loop do
134
132
  results.each { |r| yield r }
135
133
  if request_limit
136
134
  request_limit -= 1
137
- break if request_limit < 0
135
+ break if request_limit.negative?
138
136
  end
139
137
  break unless results.next?
140
138
  results = results.next
@@ -144,9 +142,7 @@ module Google
144
142
  ##
145
143
  # @private New Model::List from a response object.
146
144
  def self.from_gapi gapi_list, service, dataset_id = nil, max = nil
147
- models = List.new(Array(gapi_list[:models]).map do |gapi_json|
148
- Model.from_gapi_json gapi_json, service
149
- end)
145
+ models = List.new(Array(gapi_list[:models]).map { |gapi_json| Model.from_gapi_json gapi_json, service })
150
146
  models.instance_variable_set :@token, gapi_list[:nextPageToken]
151
147
  models.instance_variable_set :@service, service
152
148
  models.instance_variable_set :@dataset_id, dataset_id
@@ -91,8 +91,7 @@ module Google
91
91
  # @return [String] The service account email address.
92
92
  #
93
93
  def service_account_email
94
- @service_account_email ||= \
95
- service.project_service_account.email
94
+ @service_account_email ||= service.project_service_account.email
96
95
  end
97
96
 
98
97
  ##
@@ -182,11 +181,9 @@ module Google
182
181
  #
183
182
  # @!group Data
184
183
  #
185
- def copy_job source_table, destination_table, create: nil, write: nil,
186
- job_id: nil, prefix: nil, labels: nil
184
+ def copy_job source_table, destination_table, create: nil, write: nil, job_id: nil, prefix: nil, labels: nil
187
185
  ensure_service!
188
- options = { create: create, write: write, labels: labels,
189
- job_id: job_id, prefix: prefix }
186
+ options = { create: create, write: write, labels: labels, job_id: job_id, prefix: prefix }
190
187
 
191
188
  updater = CopyJob::Updater.from_options(
192
189
  service,
@@ -261,13 +258,8 @@ module Google
261
258
  #
262
259
  # @!group Data
263
260
  #
264
- def copy source_table, destination_table, create: nil, write: nil,
265
- &block
266
- job = copy_job source_table,
267
- destination_table,
268
- create: create,
269
- write: write,
270
- &block
261
+ def copy source_table, destination_table, create: nil, write: nil, &block
262
+ job = copy_job source_table, destination_table, create: create, write: write, &block
271
263
  job.wait_until_done!
272
264
  ensure_job_succeeded! job
273
265
  true
@@ -538,23 +530,16 @@ module Google
538
530
  # end
539
531
  # end
540
532
  #
541
- def query_job query, params: nil, external: nil,
542
- priority: "INTERACTIVE", cache: true, table: nil,
543
- create: nil, write: nil, dryrun: nil, dataset: nil,
544
- project: nil, standard_sql: nil, legacy_sql: nil,
545
- large_results: nil, flatten: nil,
546
- maximum_billing_tier: nil, maximum_bytes_billed: nil,
533
+ def query_job query, params: nil, external: nil, priority: "INTERACTIVE", cache: true, table: nil, create: nil,
534
+ write: nil, dryrun: nil, dataset: nil, project: nil, standard_sql: nil, legacy_sql: nil,
535
+ large_results: nil, flatten: nil, maximum_billing_tier: nil, maximum_bytes_billed: nil,
547
536
  job_id: nil, prefix: nil, labels: nil, udfs: nil
548
537
  ensure_service!
549
- options = { priority: priority, cache: cache, table: table,
550
- create: create, write: write, dryrun: dryrun,
551
- large_results: large_results, flatten: flatten,
552
- dataset: dataset, project: (project || self.project),
553
- legacy_sql: legacy_sql, standard_sql: standard_sql,
554
- maximum_billing_tier: maximum_billing_tier,
555
- maximum_bytes_billed: maximum_bytes_billed,
556
- external: external, job_id: job_id, prefix: prefix,
557
- labels: labels, udfs: udfs, params: params }
538
+ options = { priority: priority, cache: cache, table: table, create: create, write: write, dryrun: dryrun,
539
+ large_results: large_results, flatten: flatten, dataset: dataset,
540
+ project: (project || self.project), legacy_sql: legacy_sql, standard_sql: standard_sql,
541
+ maximum_billing_tier: maximum_billing_tier, maximum_bytes_billed: maximum_bytes_billed,
542
+ external: external, job_id: job_id, prefix: prefix, labels: labels, udfs: udfs, params: params }
558
543
 
559
544
  updater = QueryJob::Updater.from_options service, query, options
560
545
 
@@ -759,13 +744,10 @@ module Google
759
744
  # puts row[:name]
760
745
  # end
761
746
  #
762
- def query query, params: nil, external: nil, max: nil, cache: true,
763
- dataset: nil, project: nil, standard_sql: nil,
764
- legacy_sql: nil, &block
765
- job = query_job query, params: params, external: external,
766
- cache: cache, dataset: dataset,
767
- project: project, standard_sql: standard_sql,
768
- legacy_sql: legacy_sql, &block
747
+ def query query, params: nil, external: nil, max: nil, cache: true, dataset: nil, project: nil,
748
+ standard_sql: nil, legacy_sql: nil, &block
749
+ job = query_job query, params: params, external: external, cache: cache, dataset: dataset, project: project,
750
+ standard_sql: standard_sql, legacy_sql: legacy_sql, &block
769
751
  job.wait_until_done!
770
752
 
771
753
  if job.failed?
@@ -861,9 +843,7 @@ module Google
861
843
  #
862
844
  def dataset dataset_id, skip_lookup: nil
863
845
  ensure_service!
864
- if skip_lookup
865
- return Dataset.new_reference project, dataset_id, service
866
- end
846
+ return Dataset.new_reference project, dataset_id, service if skip_lookup
867
847
  gapi = service.get_dataset dataset_id
868
848
  Dataset.from_gapi gapi, service
869
849
  rescue Google::Cloud::NotFoundError
@@ -1092,10 +1072,8 @@ module Google
1092
1072
  def jobs all: nil, token: nil, max: nil, filter: nil,
1093
1073
  min_created_at: nil, max_created_at: nil
1094
1074
  ensure_service!
1095
- options = {
1096
- all: all, token: token, max: max, filter: filter,
1097
- min_created_at: min_created_at, max_created_at: max_created_at
1098
- }
1075
+ options = { all: all, token: token, max: max, filter: filter, min_created_at: min_created_at,
1076
+ max_created_at: max_created_at }
1099
1077
  gapi = service.list_jobs options
1100
1078
  Job::List.from_gapi gapi, service, options
1101
1079
  end
@@ -1386,17 +1364,14 @@ module Google
1386
1364
  #
1387
1365
  # @!group Data
1388
1366
  #
1389
- def extract_job table, extract_url, format: nil, compression: nil,
1390
- delimiter: nil, header: nil, job_id: nil, prefix: nil,
1391
- labels: nil
1367
+ def extract_job table, extract_url, format: nil, compression: nil, delimiter: nil, header: nil, job_id: nil,
1368
+ prefix: nil, labels: nil
1392
1369
  ensure_service!
1393
- options = { format: format, compression: compression,
1394
- delimiter: delimiter, header: header, job_id: job_id,
1370
+ options = { format: format, compression: compression, delimiter: delimiter, header: header, job_id: job_id,
1395
1371
  prefix: prefix, labels: labels }
1396
1372
 
1397
1373
  table_ref = Service.get_table_ref table, default_ref: project_ref
1398
- updater = ExtractJob::Updater.from_options service, table_ref,
1399
- extract_url, options
1374
+ updater = ExtractJob::Updater.from_options service, table_ref, extract_url, options
1400
1375
 
1401
1376
  yield updater if block_given?
1402
1377
 
@@ -1460,10 +1435,8 @@ module Google
1460
1435
  #
1461
1436
  # @!group Data
1462
1437
  #
1463
- def extract table, extract_url, format: nil, compression: nil,
1464
- delimiter: nil, header: nil, &block
1465
- job = extract_job table,
1466
- extract_url,
1438
+ def extract table, extract_url, format: nil, compression: nil, delimiter: nil, header: nil, &block
1439
+ job = extract_job table, extract_url,
1467
1440
  format: format,
1468
1441
  compression: compression,
1469
1442
  delimiter: delimiter,
@@ -1487,9 +1460,7 @@ module Google
1487
1460
 
1488
1461
  # TODO: remove `Integer` and set normally after migrating to Gax or
1489
1462
  # to google-api-client 0.10 (See google/google-api-ruby-client#439)
1490
- if gapi.numeric_id
1491
- p.instance_variable_set :@numeric_id, Integer(gapi.numeric_id)
1492
- end
1463
+ p.instance_variable_set :@numeric_id, Integer(gapi.numeric_id) if gapi.numeric_id
1493
1464
  end
1494
1465
  end
1495
1466
 
@@ -124,15 +124,13 @@ module Google
124
124
  #
125
125
  def all request_limit: nil
126
126
  request_limit = request_limit.to_i if request_limit
127
- unless block_given?
128
- return enum_for :all, request_limit: request_limit
129
- end
127
+ return enum_for :all, request_limit: request_limit unless block_given?
130
128
  results = self
131
129
  loop do
132
130
  results.each { |r| yield r }
133
131
  if request_limit
134
132
  request_limit -= 1
135
- break if request_limit < 0
133
+ break if request_limit.negative?
136
134
  end
137
135
  break unless results.next?
138
136
  results = results.next
@@ -142,9 +140,7 @@ module Google
142
140
  ##
143
141
  # @private New Project::List from a response object.
144
142
  def self.from_gapi gapi_list, service, max = nil
145
- projects = List.new(Array(gapi_list.projects).map do |gapi_object|
146
- Project.from_gapi gapi_object, service
147
- end)
143
+ projects = List.new(Array(gapi_list.projects).map { |gapi_object| Project.from_gapi gapi_object, service })
148
144
  projects.instance_variable_set :@token, gapi_list.next_page_token
149
145
  projects.instance_variable_set :@etag, gapi_list.etag
150
146
  projects.instance_variable_set :@service, service