google-cloud-bigquery 1.16.0 → 1.17.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (33) hide show
  1. checksums.yaml +4 -4
  2. data/AUTHENTICATION.md +7 -27
  3. data/CHANGELOG.md +8 -0
  4. data/CONTRIBUTING.md +1 -1
  5. data/OVERVIEW.md +5 -5
  6. data/lib/google-cloud-bigquery.rb +5 -11
  7. data/lib/google/cloud/bigquery.rb +3 -5
  8. data/lib/google/cloud/bigquery/convert.rb +23 -46
  9. data/lib/google/cloud/bigquery/copy_job.rb +6 -16
  10. data/lib/google/cloud/bigquery/credentials.rb +5 -12
  11. data/lib/google/cloud/bigquery/data.rb +10 -16
  12. data/lib/google/cloud/bigquery/dataset.rb +58 -118
  13. data/lib/google/cloud/bigquery/dataset/access.rb +5 -13
  14. data/lib/google/cloud/bigquery/dataset/list.rb +4 -9
  15. data/lib/google/cloud/bigquery/external.rb +14 -35
  16. data/lib/google/cloud/bigquery/extract_job.rb +2 -5
  17. data/lib/google/cloud/bigquery/insert_response.rb +1 -3
  18. data/lib/google/cloud/bigquery/job.rb +5 -9
  19. data/lib/google/cloud/bigquery/job/list.rb +3 -7
  20. data/lib/google/cloud/bigquery/load_job.rb +18 -33
  21. data/lib/google/cloud/bigquery/model.rb +1 -4
  22. data/lib/google/cloud/bigquery/model/list.rb +3 -7
  23. data/lib/google/cloud/bigquery/project.rb +27 -56
  24. data/lib/google/cloud/bigquery/project/list.rb +3 -7
  25. data/lib/google/cloud/bigquery/query_job.rb +40 -79
  26. data/lib/google/cloud/bigquery/schema.rb +3 -8
  27. data/lib/google/cloud/bigquery/schema/field.rb +6 -11
  28. data/lib/google/cloud/bigquery/service.rb +26 -58
  29. data/lib/google/cloud/bigquery/table.rb +58 -112
  30. data/lib/google/cloud/bigquery/table/async_inserter.rb +10 -18
  31. data/lib/google/cloud/bigquery/table/list.rb +3 -7
  32. data/lib/google/cloud/bigquery/version.rb +1 -1
  33. metadata +36 -42
@@ -57,9 +57,7 @@ module Google
57
57
  def source
58
58
  table = @gapi.configuration.extract.source_table
59
59
  return nil unless table
60
- retrieve_table table.project_id,
61
- table.dataset_id,
62
- table.table_id
60
+ retrieve_table table.project_id, table.dataset_id, table.table_id
63
61
  end
64
62
 
65
63
  ##
@@ -291,8 +289,7 @@ module Google
291
289
  # @!group Attributes
292
290
  #
293
291
  def format= new_format
294
- @gapi.configuration.extract.update! \
295
- destination_format: Convert.source_format(new_format)
292
+ @gapi.configuration.extract.update! destination_format: Convert.source_format(new_format)
296
293
  end
297
294
 
298
295
  ##
@@ -99,9 +99,7 @@ module Google
99
99
  # data.
100
100
  #
101
101
  def error_rows
102
- Array(@gapi.insert_errors).map do |ie|
103
- @rows[ie.index]
104
- end
102
+ Array(@gapi.insert_errors).map { |ie| @rows[ie.index] }
105
103
  end
106
104
 
107
105
  ##
@@ -440,15 +440,11 @@ module Google
440
440
  end
441
441
 
442
442
  def status_code_for_reason reason
443
- codes = { "accessDenied" => 403, "backendError" => 500,
444
- "billingNotEnabled" => 403,
445
- "billingTierLimitExceeded" => 400, "blocked" => 403,
446
- "duplicate" => 409, "internalError" => 500,
447
- "invalid" => 400, "invalidQuery" => 400, "notFound" => 404,
448
- "notImplemented" => 501, "quotaExceeded" => 403,
449
- "rateLimitExceeded" => 403, "resourceInUse" => 400,
450
- "resourcesExceeded" => 400, "responseTooLarge" => 403,
451
- "tableUnavailable" => 400 }
443
+ codes = { "accessDenied" => 403, "backendError" => 500, "billingNotEnabled" => 403,
444
+ "billingTierLimitExceeded" => 400, "blocked" => 403, "duplicate" => 409, "internalError" => 500,
445
+ "invalid" => 400, "invalidQuery" => 400, "notFound" => 404, "notImplemented" => 501,
446
+ "quotaExceeded" => 403, "rateLimitExceeded" => 403, "resourceInUse" => 400,
447
+ "resourcesExceeded" => 400, "responseTooLarge" => 403, "tableUnavailable" => 400 }
452
448
  codes[reason] || 0
453
449
  end
454
450
  end
@@ -123,15 +123,13 @@ module Google
123
123
  #
124
124
  def all request_limit: nil
125
125
  request_limit = request_limit.to_i if request_limit
126
- unless block_given?
127
- return enum_for :all, request_limit: request_limit
128
- end
126
+ return enum_for :all, request_limit: request_limit unless block_given?
129
127
  results = self
130
128
  loop do
131
129
  results.each { |r| yield r }
132
130
  if request_limit
133
131
  request_limit -= 1
134
- break if request_limit < 0
132
+ break if request_limit.negative?
135
133
  end
136
134
  break unless results.next?
137
135
  results = results.next
@@ -142,9 +140,7 @@ module Google
142
140
  # @private New Job::List from a Google API Client
143
141
  # Google::Apis::BigqueryV2::JobList object.
144
142
  def self.from_gapi gapi_list, service, options = {}
145
- jobs = List.new(Array(gapi_list.jobs).map do |gapi_object|
146
- Job.from_gapi gapi_object, service
147
- end)
143
+ jobs = List.new(Array(gapi_list.jobs).map { |gapi_object| Job.from_gapi gapi_object, service })
148
144
  jobs.instance_variable_set :@token, gapi_list.next_page_token
149
145
  jobs.instance_variable_set :@etag, gapi_list.etag
150
146
  jobs.instance_variable_set :@service, service
@@ -67,9 +67,7 @@ module Google
67
67
  def destination
68
68
  table = @gapi.configuration.load.destination_table
69
69
  return nil unless table
70
- retrieve_table table.project_id,
71
- table.dataset_id,
72
- table.table_id
70
+ retrieve_table table.project_id, table.dataset_id, table.table_id
73
71
  end
74
72
 
75
73
  ##
@@ -402,9 +400,10 @@ module Google
402
400
  # @!group Attributes
403
401
  #
404
402
  def time_partitioning_expiration
405
- @gapi.configuration.load.time_partitioning.expiration_ms / 1_000 if
406
- time_partitioning? &&
407
- !@gapi.configuration.load.time_partitioning.expiration_ms.nil?
403
+ return nil unless time_partitioning?
404
+ return nil if @gapi.configuration.load.time_partitioning.expiration_ms.nil?
405
+
406
+ @gapi.configuration.load.time_partitioning.expiration_ms / 1_000
408
407
  end
409
408
 
410
409
  ###
@@ -934,8 +933,7 @@ module Google
934
933
  # @!group Attributes
935
934
  #
936
935
  def format= new_format
937
- @gapi.configuration.load.update! \
938
- source_format: Convert.source_format(new_format)
936
+ @gapi.configuration.load.update! source_format: Convert.source_format(new_format)
939
937
  end
940
938
 
941
939
  ##
@@ -955,8 +953,7 @@ module Google
955
953
  # @!group Attributes
956
954
  #
957
955
  def create= new_create
958
- @gapi.configuration.load.update! \
959
- create_disposition: Convert.create_disposition(new_create)
956
+ @gapi.configuration.load.update! create_disposition: Convert.create_disposition(new_create)
960
957
  end
961
958
 
962
959
  ##
@@ -977,8 +974,7 @@ module Google
977
974
  # @!group Attributes
978
975
  #
979
976
  def write= new_write
980
- @gapi.configuration.load.update! \
981
- write_disposition: Convert.write_disposition(new_write)
977
+ @gapi.configuration.load.update! write_disposition: Convert.write_disposition(new_write)
982
978
  end
983
979
 
984
980
  ##
@@ -999,8 +995,7 @@ module Google
999
995
  if new_fields.nil?
1000
996
  @gapi.configuration.load.update! projection_fields: nil
1001
997
  else
1002
- @gapi.configuration.load.update! \
1003
- projection_fields: Array(new_fields)
998
+ @gapi.configuration.load.update! projection_fields: Array(new_fields)
1004
999
  end
1005
1000
  end
1006
1001
 
@@ -1199,8 +1194,7 @@ module Google
1199
1194
  if new_options.nil?
1200
1195
  @gapi.configuration.load.update! schema_update_options: nil
1201
1196
  else
1202
- @gapi.configuration.load.update! \
1203
- schema_update_options: Array(new_options)
1197
+ @gapi.configuration.load.update! schema_update_options: Array(new_options)
1204
1198
  end
1205
1199
  end
1206
1200
 
@@ -1238,9 +1232,7 @@ module Google
1238
1232
  #
1239
1233
  # @!group Attributes
1240
1234
  def encryption= val
1241
- @gapi.configuration.load.update!(
1242
- destination_encryption_configuration: val.to_gapi
1243
- )
1235
+ @gapi.configuration.load.update! destination_encryption_configuration: val.to_gapi
1244
1236
  end
1245
1237
 
1246
1238
  ##
@@ -1288,8 +1280,7 @@ module Google
1288
1280
  # @!group Attributes
1289
1281
  #
1290
1282
  def time_partitioning_type= type
1291
- @gapi.configuration.load.time_partitioning ||= \
1292
- Google::Apis::BigqueryV2::TimePartitioning.new
1283
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
1293
1284
  @gapi.configuration.load.time_partitioning.update! type: type
1294
1285
  end
1295
1286
 
@@ -1332,8 +1323,7 @@ module Google
1332
1323
  # @!group Attributes
1333
1324
  #
1334
1325
  def time_partitioning_field= field
1335
- @gapi.configuration.load.time_partitioning ||= \
1336
- Google::Apis::BigqueryV2::TimePartitioning.new
1326
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
1337
1327
  @gapi.configuration.load.time_partitioning.update! field: field
1338
1328
  end
1339
1329
 
@@ -1366,10 +1356,8 @@ module Google
1366
1356
  # @!group Attributes
1367
1357
  #
1368
1358
  def time_partitioning_expiration= expiration
1369
- @gapi.configuration.load.time_partitioning ||= \
1370
- Google::Apis::BigqueryV2::TimePartitioning.new
1371
- @gapi.configuration.load.time_partitioning.update! \
1372
- expiration_ms: expiration * 1000
1359
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
1360
+ @gapi.configuration.load.time_partitioning.update! expiration_ms: expiration * 1000
1373
1361
  end
1374
1362
 
1375
1363
  ##
@@ -1384,10 +1372,8 @@ module Google
1384
1372
  # @!group Attributes
1385
1373
  #
1386
1374
  def time_partitioning_require_filter= val
1387
- @gapi.configuration.load.time_partitioning ||= \
1388
- Google::Apis::BigqueryV2::TimePartitioning.new
1389
- @gapi.configuration.load.time_partitioning.update! \
1390
- require_partition_filter: val
1375
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
1376
+ @gapi.configuration.load.time_partitioning.update! require_partition_filter: val
1391
1377
  end
1392
1378
 
1393
1379
  ##
@@ -1436,8 +1422,7 @@ module Google
1436
1422
  # @!group Attributes
1437
1423
  #
1438
1424
  def clustering_fields= fields
1439
- @gapi.configuration.load.clustering ||= \
1440
- Google::Apis::BigqueryV2::Clustering.new
1425
+ @gapi.configuration.load.clustering ||= Google::Apis::BigqueryV2::Clustering.new
1441
1426
  @gapi.configuration.load.clustering.fields = fields
1442
1427
  end
1443
1428
 
@@ -397,10 +397,7 @@ module Google
397
397
  # We have to create a gapic object from the hash because that is what
398
398
  # EncryptionConfiguration is expecing.
399
399
  json_cmek = @gapi_json[:encryptionConfiguration].to_json
400
- gapi_cmek = \
401
- Google::Apis::BigqueryV2::EncryptionConfiguration.from_json(
402
- json_cmek
403
- )
400
+ gapi_cmek = Google::Apis::BigqueryV2::EncryptionConfiguration.from_json json_cmek
404
401
  EncryptionConfiguration.from_gapi(gapi_cmek).freeze
405
402
  end
406
403
 
@@ -126,15 +126,13 @@ module Google
126
126
  #
127
127
  def all request_limit: nil
128
128
  request_limit = request_limit.to_i if request_limit
129
- unless block_given?
130
- return enum_for :all, request_limit: request_limit
131
- end
129
+ return enum_for :all, request_limit: request_limit unless block_given?
132
130
  results = self
133
131
  loop do
134
132
  results.each { |r| yield r }
135
133
  if request_limit
136
134
  request_limit -= 1
137
- break if request_limit < 0
135
+ break if request_limit.negative?
138
136
  end
139
137
  break unless results.next?
140
138
  results = results.next
@@ -144,9 +142,7 @@ module Google
144
142
  ##
145
143
  # @private New Model::List from a response object.
146
144
  def self.from_gapi gapi_list, service, dataset_id = nil, max = nil
147
- models = List.new(Array(gapi_list[:models]).map do |gapi_json|
148
- Model.from_gapi_json gapi_json, service
149
- end)
145
+ models = List.new(Array(gapi_list[:models]).map { |gapi_json| Model.from_gapi_json gapi_json, service })
150
146
  models.instance_variable_set :@token, gapi_list[:nextPageToken]
151
147
  models.instance_variable_set :@service, service
152
148
  models.instance_variable_set :@dataset_id, dataset_id
@@ -91,8 +91,7 @@ module Google
91
91
  # @return [String] The service account email address.
92
92
  #
93
93
  def service_account_email
94
- @service_account_email ||= \
95
- service.project_service_account.email
94
+ @service_account_email ||= service.project_service_account.email
96
95
  end
97
96
 
98
97
  ##
@@ -182,11 +181,9 @@ module Google
182
181
  #
183
182
  # @!group Data
184
183
  #
185
- def copy_job source_table, destination_table, create: nil, write: nil,
186
- job_id: nil, prefix: nil, labels: nil
184
+ def copy_job source_table, destination_table, create: nil, write: nil, job_id: nil, prefix: nil, labels: nil
187
185
  ensure_service!
188
- options = { create: create, write: write, labels: labels,
189
- job_id: job_id, prefix: prefix }
186
+ options = { create: create, write: write, labels: labels, job_id: job_id, prefix: prefix }
190
187
 
191
188
  updater = CopyJob::Updater.from_options(
192
189
  service,
@@ -261,13 +258,8 @@ module Google
261
258
  #
262
259
  # @!group Data
263
260
  #
264
- def copy source_table, destination_table, create: nil, write: nil,
265
- &block
266
- job = copy_job source_table,
267
- destination_table,
268
- create: create,
269
- write: write,
270
- &block
261
+ def copy source_table, destination_table, create: nil, write: nil, &block
262
+ job = copy_job source_table, destination_table, create: create, write: write, &block
271
263
  job.wait_until_done!
272
264
  ensure_job_succeeded! job
273
265
  true
@@ -538,23 +530,16 @@ module Google
538
530
  # end
539
531
  # end
540
532
  #
541
- def query_job query, params: nil, external: nil,
542
- priority: "INTERACTIVE", cache: true, table: nil,
543
- create: nil, write: nil, dryrun: nil, dataset: nil,
544
- project: nil, standard_sql: nil, legacy_sql: nil,
545
- large_results: nil, flatten: nil,
546
- maximum_billing_tier: nil, maximum_bytes_billed: nil,
533
+ def query_job query, params: nil, external: nil, priority: "INTERACTIVE", cache: true, table: nil, create: nil,
534
+ write: nil, dryrun: nil, dataset: nil, project: nil, standard_sql: nil, legacy_sql: nil,
535
+ large_results: nil, flatten: nil, maximum_billing_tier: nil, maximum_bytes_billed: nil,
547
536
  job_id: nil, prefix: nil, labels: nil, udfs: nil
548
537
  ensure_service!
549
- options = { priority: priority, cache: cache, table: table,
550
- create: create, write: write, dryrun: dryrun,
551
- large_results: large_results, flatten: flatten,
552
- dataset: dataset, project: (project || self.project),
553
- legacy_sql: legacy_sql, standard_sql: standard_sql,
554
- maximum_billing_tier: maximum_billing_tier,
555
- maximum_bytes_billed: maximum_bytes_billed,
556
- external: external, job_id: job_id, prefix: prefix,
557
- labels: labels, udfs: udfs, params: params }
538
+ options = { priority: priority, cache: cache, table: table, create: create, write: write, dryrun: dryrun,
539
+ large_results: large_results, flatten: flatten, dataset: dataset,
540
+ project: (project || self.project), legacy_sql: legacy_sql, standard_sql: standard_sql,
541
+ maximum_billing_tier: maximum_billing_tier, maximum_bytes_billed: maximum_bytes_billed,
542
+ external: external, job_id: job_id, prefix: prefix, labels: labels, udfs: udfs, params: params }
558
543
 
559
544
  updater = QueryJob::Updater.from_options service, query, options
560
545
 
@@ -759,13 +744,10 @@ module Google
759
744
  # puts row[:name]
760
745
  # end
761
746
  #
762
- def query query, params: nil, external: nil, max: nil, cache: true,
763
- dataset: nil, project: nil, standard_sql: nil,
764
- legacy_sql: nil, &block
765
- job = query_job query, params: params, external: external,
766
- cache: cache, dataset: dataset,
767
- project: project, standard_sql: standard_sql,
768
- legacy_sql: legacy_sql, &block
747
+ def query query, params: nil, external: nil, max: nil, cache: true, dataset: nil, project: nil,
748
+ standard_sql: nil, legacy_sql: nil, &block
749
+ job = query_job query, params: params, external: external, cache: cache, dataset: dataset, project: project,
750
+ standard_sql: standard_sql, legacy_sql: legacy_sql, &block
769
751
  job.wait_until_done!
770
752
 
771
753
  if job.failed?
@@ -861,9 +843,7 @@ module Google
861
843
  #
862
844
  def dataset dataset_id, skip_lookup: nil
863
845
  ensure_service!
864
- if skip_lookup
865
- return Dataset.new_reference project, dataset_id, service
866
- end
846
+ return Dataset.new_reference project, dataset_id, service if skip_lookup
867
847
  gapi = service.get_dataset dataset_id
868
848
  Dataset.from_gapi gapi, service
869
849
  rescue Google::Cloud::NotFoundError
@@ -1092,10 +1072,8 @@ module Google
1092
1072
  def jobs all: nil, token: nil, max: nil, filter: nil,
1093
1073
  min_created_at: nil, max_created_at: nil
1094
1074
  ensure_service!
1095
- options = {
1096
- all: all, token: token, max: max, filter: filter,
1097
- min_created_at: min_created_at, max_created_at: max_created_at
1098
- }
1075
+ options = { all: all, token: token, max: max, filter: filter, min_created_at: min_created_at,
1076
+ max_created_at: max_created_at }
1099
1077
  gapi = service.list_jobs options
1100
1078
  Job::List.from_gapi gapi, service, options
1101
1079
  end
@@ -1386,17 +1364,14 @@ module Google
1386
1364
  #
1387
1365
  # @!group Data
1388
1366
  #
1389
- def extract_job table, extract_url, format: nil, compression: nil,
1390
- delimiter: nil, header: nil, job_id: nil, prefix: nil,
1391
- labels: nil
1367
+ def extract_job table, extract_url, format: nil, compression: nil, delimiter: nil, header: nil, job_id: nil,
1368
+ prefix: nil, labels: nil
1392
1369
  ensure_service!
1393
- options = { format: format, compression: compression,
1394
- delimiter: delimiter, header: header, job_id: job_id,
1370
+ options = { format: format, compression: compression, delimiter: delimiter, header: header, job_id: job_id,
1395
1371
  prefix: prefix, labels: labels }
1396
1372
 
1397
1373
  table_ref = Service.get_table_ref table, default_ref: project_ref
1398
- updater = ExtractJob::Updater.from_options service, table_ref,
1399
- extract_url, options
1374
+ updater = ExtractJob::Updater.from_options service, table_ref, extract_url, options
1400
1375
 
1401
1376
  yield updater if block_given?
1402
1377
 
@@ -1460,10 +1435,8 @@ module Google
1460
1435
  #
1461
1436
  # @!group Data
1462
1437
  #
1463
- def extract table, extract_url, format: nil, compression: nil,
1464
- delimiter: nil, header: nil, &block
1465
- job = extract_job table,
1466
- extract_url,
1438
+ def extract table, extract_url, format: nil, compression: nil, delimiter: nil, header: nil, &block
1439
+ job = extract_job table, extract_url,
1467
1440
  format: format,
1468
1441
  compression: compression,
1469
1442
  delimiter: delimiter,
@@ -1487,9 +1460,7 @@ module Google
1487
1460
 
1488
1461
  # TODO: remove `Integer` and set normally after migrating to Gax or
1489
1462
  # to google-api-client 0.10 (See google/google-api-ruby-client#439)
1490
- if gapi.numeric_id
1491
- p.instance_variable_set :@numeric_id, Integer(gapi.numeric_id)
1492
- end
1463
+ p.instance_variable_set :@numeric_id, Integer(gapi.numeric_id) if gapi.numeric_id
1493
1464
  end
1494
1465
  end
1495
1466
 
@@ -124,15 +124,13 @@ module Google
124
124
  #
125
125
  def all request_limit: nil
126
126
  request_limit = request_limit.to_i if request_limit
127
- unless block_given?
128
- return enum_for :all, request_limit: request_limit
129
- end
127
+ return enum_for :all, request_limit: request_limit unless block_given?
130
128
  results = self
131
129
  loop do
132
130
  results.each { |r| yield r }
133
131
  if request_limit
134
132
  request_limit -= 1
135
- break if request_limit < 0
133
+ break if request_limit.negative?
136
134
  end
137
135
  break unless results.next?
138
136
  results = results.next
@@ -142,9 +140,7 @@ module Google
142
140
  ##
143
141
  # @private New Project::List from a response object.
144
142
  def self.from_gapi gapi_list, service, max = nil
145
- projects = List.new(Array(gapi_list.projects).map do |gapi_object|
146
- Project.from_gapi gapi_object, service
147
- end)
143
+ projects = List.new(Array(gapi_list.projects).map { |gapi_object| Project.from_gapi gapi_object, service })
148
144
  projects.instance_variable_set :@token, gapi_list.next_page_token
149
145
  projects.instance_variable_set :@etag, gapi_list.etag
150
146
  projects.instance_variable_set :@service, service