google-cloud-bigquery 1.14.0 → 1.42.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/AUTHENTICATION.md +17 -54
- data/CHANGELOG.md +377 -0
- data/CONTRIBUTING.md +328 -116
- data/LOGGING.md +1 -1
- data/OVERVIEW.md +21 -20
- data/TROUBLESHOOTING.md +2 -8
- data/lib/google/cloud/bigquery/argument.rb +197 -0
- data/lib/google/cloud/bigquery/convert.rb +155 -173
- data/lib/google/cloud/bigquery/copy_job.rb +74 -26
- data/lib/google/cloud/bigquery/credentials.rb +5 -12
- data/lib/google/cloud/bigquery/data.rb +109 -18
- data/lib/google/cloud/bigquery/dataset/access.rb +474 -52
- data/lib/google/cloud/bigquery/dataset/list.rb +7 -13
- data/lib/google/cloud/bigquery/dataset/tag.rb +67 -0
- data/lib/google/cloud/bigquery/dataset.rb +1044 -287
- data/lib/google/cloud/bigquery/external/avro_source.rb +107 -0
- data/lib/google/cloud/bigquery/external/bigtable_source/column.rb +404 -0
- data/lib/google/cloud/bigquery/external/bigtable_source/column_family.rb +945 -0
- data/lib/google/cloud/bigquery/external/bigtable_source.rb +230 -0
- data/lib/google/cloud/bigquery/external/csv_source.rb +481 -0
- data/lib/google/cloud/bigquery/external/data_source.rb +771 -0
- data/lib/google/cloud/bigquery/external/json_source.rb +170 -0
- data/lib/google/cloud/bigquery/external/parquet_source.rb +148 -0
- data/lib/google/cloud/bigquery/external/sheets_source.rb +166 -0
- data/lib/google/cloud/bigquery/external.rb +50 -2256
- data/lib/google/cloud/bigquery/extract_job.rb +226 -61
- data/lib/google/cloud/bigquery/insert_response.rb +1 -3
- data/lib/google/cloud/bigquery/job/list.rb +10 -14
- data/lib/google/cloud/bigquery/job.rb +289 -14
- data/lib/google/cloud/bigquery/load_job.rb +810 -136
- data/lib/google/cloud/bigquery/model/list.rb +5 -9
- data/lib/google/cloud/bigquery/model.rb +247 -16
- data/lib/google/cloud/bigquery/policy.rb +432 -0
- data/lib/google/cloud/bigquery/project/list.rb +6 -11
- data/lib/google/cloud/bigquery/project.rb +509 -250
- data/lib/google/cloud/bigquery/query_job.rb +594 -128
- data/lib/google/cloud/bigquery/routine/list.rb +165 -0
- data/lib/google/cloud/bigquery/routine.rb +1227 -0
- data/lib/google/cloud/bigquery/schema/field.rb +413 -63
- data/lib/google/cloud/bigquery/schema.rb +221 -48
- data/lib/google/cloud/bigquery/service.rb +204 -112
- data/lib/google/cloud/bigquery/standard_sql.rb +269 -53
- data/lib/google/cloud/bigquery/table/async_inserter.rb +86 -43
- data/lib/google/cloud/bigquery/table/list.rb +6 -11
- data/lib/google/cloud/bigquery/table.rb +1470 -377
- data/lib/google/cloud/bigquery/time.rb +6 -0
- data/lib/google/cloud/bigquery/version.rb +1 -1
- data/lib/google/cloud/bigquery.rb +4 -6
- data/lib/google-cloud-bigquery.rb +14 -13
- metadata +66 -38
@@ -43,13 +43,13 @@ module Google
|
|
43
43
|
|
44
44
|
##
|
45
45
|
# Creates a new Service instance.
|
46
|
-
def initialize project, credentials,
|
47
|
-
retries: nil, timeout: nil, host: nil
|
46
|
+
def initialize project, credentials, retries: nil, timeout: nil, host: nil, quota_project: nil
|
48
47
|
@project = project
|
49
48
|
@credentials = credentials
|
50
49
|
@retries = retries
|
51
50
|
@timeout = timeout
|
52
51
|
@host = host
|
52
|
+
@quota_project = quota_project
|
53
53
|
end
|
54
54
|
|
55
55
|
def service
|
@@ -57,8 +57,7 @@ module Google
|
|
57
57
|
@service ||= begin
|
58
58
|
service = API::BigqueryService.new
|
59
59
|
service.client_options.application_name = "gcloud-ruby"
|
60
|
-
service.client_options.application_version =
|
61
|
-
Google::Cloud::Bigquery::VERSION
|
60
|
+
service.client_options.application_version = Google::Cloud::Bigquery::VERSION
|
62
61
|
service.client_options.open_timeout_sec = timeout
|
63
62
|
service.client_options.read_timeout_sec = timeout
|
64
63
|
service.client_options.send_timeout_sec = timeout
|
@@ -66,6 +65,9 @@ module Google
|
|
66
65
|
service.request_options.header ||= {}
|
67
66
|
service.request_options.header["x-goog-api-client"] = \
|
68
67
|
"gl-ruby/#{RUBY_VERSION} gccl/#{Google::Cloud::Bigquery::VERSION}"
|
68
|
+
service.request_options.query ||= {}
|
69
|
+
service.request_options.query["prettyPrint"] = false
|
70
|
+
service.request_options.quota_project = @quota_project if @quota_project
|
69
71
|
service.authorization = @credentials.client
|
70
72
|
service.root_url = host if host
|
71
73
|
service
|
@@ -80,12 +82,10 @@ module Google
|
|
80
82
|
##
|
81
83
|
# Lists all datasets in the specified project to which you have
|
82
84
|
# been granted the READER dataset role.
|
83
|
-
def list_datasets
|
85
|
+
def list_datasets all: nil, filter: nil, max: nil, token: nil
|
84
86
|
# The list operation is considered idempotent
|
85
87
|
execute backoff: true do
|
86
|
-
service.list_datasets
|
87
|
-
@project, all: options[:all], filter: options[:filter],
|
88
|
-
max_results: options[:max], page_token: options[:token]
|
88
|
+
service.list_datasets @project, all: all, filter: filter, max_results: max, page_token: token
|
89
89
|
end
|
90
90
|
end
|
91
91
|
|
@@ -116,8 +116,7 @@ module Google
|
|
116
116
|
patch_with_backoff = true
|
117
117
|
end
|
118
118
|
execute backoff: patch_with_backoff do
|
119
|
-
service.patch_dataset @project, dataset_id, patched_dataset_gapi,
|
120
|
-
options: options
|
119
|
+
service.patch_dataset @project, dataset_id, patched_dataset_gapi, options: options
|
121
120
|
end
|
122
121
|
end
|
123
122
|
|
@@ -136,19 +135,20 @@ module Google
|
|
136
135
|
##
|
137
136
|
# Lists all tables in the specified dataset.
|
138
137
|
# Requires the READER dataset role.
|
139
|
-
def list_tables dataset_id,
|
138
|
+
def list_tables dataset_id, max: nil, token: nil
|
140
139
|
# The list operation is considered idempotent
|
141
140
|
execute backoff: true do
|
142
|
-
service.list_tables @project, dataset_id,
|
143
|
-
max_results: options[:max],
|
144
|
-
page_token: options[:token]
|
141
|
+
service.list_tables @project, dataset_id, max_results: max, page_token: token
|
145
142
|
end
|
146
143
|
end
|
147
144
|
|
148
|
-
|
145
|
+
##
|
146
|
+
# Gets the specified table resource by full table reference.
|
147
|
+
def get_project_table project_id, dataset_id, table_id, metadata_view: nil
|
148
|
+
metadata_view = table_metadata_view_type_for metadata_view
|
149
149
|
# The get operation is considered idempotent
|
150
150
|
execute backoff: true do
|
151
|
-
service.get_table project_id, dataset_id, table_id
|
151
|
+
service.get_table project_id, dataset_id, table_id, view: metadata_view
|
152
152
|
end
|
153
153
|
end
|
154
154
|
|
@@ -157,11 +157,8 @@ module Google
|
|
157
157
|
# This method does not return the data in the table,
|
158
158
|
# it only returns the table resource,
|
159
159
|
# which describes the structure of this table.
|
160
|
-
def get_table dataset_id, table_id
|
161
|
-
|
162
|
-
execute backoff: true do
|
163
|
-
get_project_table @project, dataset_id, table_id
|
164
|
-
end
|
160
|
+
def get_table dataset_id, table_id, metadata_view: nil
|
161
|
+
get_project_table @project, dataset_id, table_id, metadata_view: metadata_view
|
165
162
|
end
|
166
163
|
|
167
164
|
##
|
@@ -182,8 +179,35 @@ module Google
|
|
182
179
|
patch_with_backoff = true
|
183
180
|
end
|
184
181
|
execute backoff: patch_with_backoff do
|
185
|
-
service.patch_table @project, dataset_id, table_id,
|
186
|
-
|
182
|
+
service.patch_table @project, dataset_id, table_id, patched_table_gapi, options: options
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
##
|
187
|
+
# Returns Google::Apis::BigqueryV2::Policy
|
188
|
+
def get_table_policy dataset_id, table_id
|
189
|
+
policy_options = API::GetPolicyOptions.new requested_policy_version: 1
|
190
|
+
execute do
|
191
|
+
service.get_table_iam_policy table_path(dataset_id, table_id),
|
192
|
+
API::GetIamPolicyRequest.new(options: policy_options)
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
##
|
197
|
+
# @param [Google::Apis::BigqueryV2::Policy] new_policy
|
198
|
+
def set_table_policy dataset_id, table_id, new_policy
|
199
|
+
execute do
|
200
|
+
service.set_table_iam_policy table_path(dataset_id, table_id),
|
201
|
+
API::SetIamPolicyRequest.new(policy: new_policy)
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
##
|
206
|
+
# Returns Google::Apis::BigqueryV2::TestIamPermissionsResponse
|
207
|
+
def test_table_permissions dataset_id, table_id, permissions
|
208
|
+
execute do
|
209
|
+
service.test_table_iam_permissions table_path(dataset_id, table_id),
|
210
|
+
API::TestIamPermissionsRequest.new(permissions: permissions)
|
187
211
|
end
|
188
212
|
end
|
189
213
|
|
@@ -196,40 +220,45 @@ module Google
|
|
196
220
|
|
197
221
|
##
|
198
222
|
# Retrieves data from the table.
|
199
|
-
def list_tabledata dataset_id, table_id,
|
223
|
+
def list_tabledata dataset_id, table_id, max: nil, token: nil, start: nil
|
200
224
|
# The list operation is considered idempotent
|
201
225
|
execute backoff: true do
|
202
226
|
json_txt = service.list_table_data \
|
203
227
|
@project, dataset_id, table_id,
|
204
|
-
max_results:
|
205
|
-
page_token:
|
206
|
-
start_index:
|
228
|
+
max_results: max,
|
229
|
+
page_token: token,
|
230
|
+
start_index: start,
|
207
231
|
options: { skip_deserialization: true }
|
208
232
|
JSON.parse json_txt, symbolize_names: true
|
209
233
|
end
|
210
234
|
end
|
211
235
|
|
212
|
-
def insert_tabledata dataset_id, table_id, rows,
|
236
|
+
def insert_tabledata dataset_id, table_id, rows, insert_ids: nil, ignore_unknown: nil, skip_invalid: nil
|
213
237
|
json_rows = Array(rows).map { |row| Convert.to_json_row row }
|
214
|
-
insert_tabledata_json_rows dataset_id, table_id, json_rows,
|
238
|
+
insert_tabledata_json_rows dataset_id, table_id, json_rows, insert_ids: insert_ids,
|
239
|
+
ignore_unknown: ignore_unknown,
|
240
|
+
skip_invalid: skip_invalid
|
215
241
|
end
|
216
242
|
|
217
|
-
def insert_tabledata_json_rows dataset_id, table_id, json_rows,
|
218
|
-
|
219
|
-
|
220
|
-
rows_and_ids = Array(json_rows).zip Array(options[:insert_ids])
|
243
|
+
def insert_tabledata_json_rows dataset_id, table_id, json_rows, insert_ids: nil, ignore_unknown: nil,
|
244
|
+
skip_invalid: nil
|
245
|
+
rows_and_ids = Array(json_rows).zip Array(insert_ids)
|
221
246
|
insert_rows = rows_and_ids.map do |json_row, insert_id|
|
222
|
-
insert_id
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
247
|
+
if insert_id == :skip
|
248
|
+
{ json: json_row }
|
249
|
+
else
|
250
|
+
insert_id ||= SecureRandom.uuid
|
251
|
+
{
|
252
|
+
insertId: insert_id,
|
253
|
+
json: json_row
|
254
|
+
}
|
255
|
+
end
|
227
256
|
end
|
228
257
|
|
229
258
|
insert_req = {
|
230
259
|
rows: insert_rows,
|
231
|
-
ignoreUnknownValues:
|
232
|
-
skipInvalidRows:
|
260
|
+
ignoreUnknownValues: ignore_unknown,
|
261
|
+
skipInvalidRows: skip_invalid
|
233
262
|
}.to_json
|
234
263
|
|
235
264
|
# The insertAll with insertId operation is considered idempotent
|
@@ -248,27 +277,26 @@ module Google
|
|
248
277
|
options = { skip_deserialization: true }
|
249
278
|
# The list operation is considered idempotent
|
250
279
|
execute backoff: true do
|
251
|
-
json_txt = service.list_models @project, dataset_id,
|
252
|
-
max_results: max,
|
253
|
-
page_token: token,
|
254
|
-
options: options
|
280
|
+
json_txt = service.list_models @project, dataset_id, max_results: max, page_token: token, options: options
|
255
281
|
JSON.parse json_txt, symbolize_names: true
|
256
282
|
end
|
257
283
|
end
|
258
284
|
|
259
|
-
# Gets the specified model resource by model
|
260
|
-
|
261
|
-
# it only returns the model resource,
|
262
|
-
# which describes the structure of this model.
|
263
|
-
def get_model dataset_id, model_id
|
285
|
+
# Gets the specified model resource by full model reference.
|
286
|
+
def get_project_model project_id, dataset_id, model_id
|
264
287
|
# The get operation is considered idempotent
|
265
288
|
execute backoff: true do
|
266
|
-
json_txt = service.get_model
|
267
|
-
options: { skip_deserialization: true }
|
289
|
+
json_txt = service.get_model project_id, dataset_id, model_id, options: { skip_deserialization: true }
|
268
290
|
JSON.parse json_txt, symbolize_names: true
|
269
291
|
end
|
270
292
|
end
|
271
293
|
|
294
|
+
# Gets the specified model resource by model ID. This method does not return the data in the model, it only
|
295
|
+
# returns the model resource, which describes the structure of this model.
|
296
|
+
def get_model dataset_id, model_id
|
297
|
+
get_project_model @project, dataset_id, model_id
|
298
|
+
end
|
299
|
+
|
272
300
|
##
|
273
301
|
# Updates information in an existing model, replacing fields that
|
274
302
|
# are provided in the submitted model resource.
|
@@ -281,9 +309,7 @@ module Google
|
|
281
309
|
patch_with_backoff = true
|
282
310
|
end
|
283
311
|
execute backoff: patch_with_backoff do
|
284
|
-
json_txt = service.patch_model @project, dataset_id, model_id,
|
285
|
-
patched_model_gapi,
|
286
|
-
options: options
|
312
|
+
json_txt = service.patch_model @project, dataset_id, model_id, patched_model_gapi, options: options
|
287
313
|
JSON.parse json_txt, symbolize_names: true
|
288
314
|
end
|
289
315
|
end
|
@@ -295,20 +321,69 @@ module Google
|
|
295
321
|
execute { service.delete_model @project, dataset_id, model_id }
|
296
322
|
end
|
297
323
|
|
324
|
+
##
|
325
|
+
# Creates a new routine in the dataset.
|
326
|
+
def insert_routine dataset_id, new_routine_gapi
|
327
|
+
execute { service.insert_routine @project, dataset_id, new_routine_gapi }
|
328
|
+
end
|
329
|
+
|
330
|
+
##
|
331
|
+
# Lists all routines in the specified dataset.
|
332
|
+
# Requires the READER dataset role.
|
333
|
+
# Unless readMask is set in the request, only the following fields are populated:
|
334
|
+
# etag, projectId, datasetId, routineId, routineType, creationTime, lastModifiedTime, and language.
|
335
|
+
def list_routines dataset_id, max: nil, token: nil, filter: nil
|
336
|
+
# The list operation is considered idempotent
|
337
|
+
execute backoff: true do
|
338
|
+
service.list_routines @project, dataset_id, max_results: max,
|
339
|
+
page_token: token,
|
340
|
+
filter: filter
|
341
|
+
end
|
342
|
+
end
|
343
|
+
|
344
|
+
##
|
345
|
+
# Gets the specified routine resource by routine ID.
|
346
|
+
def get_routine dataset_id, routine_id
|
347
|
+
# The get operation is considered idempotent
|
348
|
+
execute backoff: true do
|
349
|
+
service.get_routine @project, dataset_id, routine_id
|
350
|
+
end
|
351
|
+
end
|
352
|
+
|
353
|
+
##
|
354
|
+
# Updates information in an existing routine, replacing the entire routine resource.
|
355
|
+
def update_routine dataset_id, routine_id, new_routine_gapi
|
356
|
+
update_with_backoff = false
|
357
|
+
options = {}
|
358
|
+
if new_routine_gapi.etag
|
359
|
+
options[:header] = { "If-Match" => new_routine_gapi.etag }
|
360
|
+
# The update with etag operation is considered idempotent
|
361
|
+
update_with_backoff = true
|
362
|
+
end
|
363
|
+
execute backoff: update_with_backoff do
|
364
|
+
service.update_routine @project, dataset_id, routine_id, new_routine_gapi, options: options
|
365
|
+
end
|
366
|
+
end
|
367
|
+
|
368
|
+
##
|
369
|
+
# Deletes the routine specified by routine_id from the dataset.
|
370
|
+
def delete_routine dataset_id, routine_id
|
371
|
+
execute { service.delete_routine @project, dataset_id, routine_id }
|
372
|
+
end
|
373
|
+
|
298
374
|
##
|
299
375
|
# Lists all jobs in the specified project to which you have
|
300
376
|
# been granted the READER job role.
|
301
|
-
def list_jobs
|
377
|
+
def list_jobs all: nil, token: nil, max: nil, filter: nil, min_created_at: nil, max_created_at: nil,
|
378
|
+
parent_job_id: nil
|
302
379
|
# The list operation is considered idempotent
|
303
|
-
min_creation_time = Convert.time_to_millis
|
304
|
-
max_creation_time = Convert.time_to_millis
|
380
|
+
min_creation_time = Convert.time_to_millis min_created_at
|
381
|
+
max_creation_time = Convert.time_to_millis max_created_at
|
305
382
|
execute backoff: true do
|
306
|
-
service.list_jobs
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
min_creation_time: min_creation_time,
|
311
|
-
max_creation_time: max_creation_time
|
383
|
+
service.list_jobs @project, all_users: all, max_results: max,
|
384
|
+
page_token: token, projection: "full", state_filter: filter,
|
385
|
+
min_creation_time: min_creation_time, max_creation_time: max_creation_time,
|
386
|
+
parent_job_id: parent_job_id
|
312
387
|
end
|
313
388
|
end
|
314
389
|
|
@@ -331,10 +406,7 @@ module Google
|
|
331
406
|
end
|
332
407
|
|
333
408
|
def insert_job config, location: nil
|
334
|
-
job_object = API::Job.new(
|
335
|
-
job_reference: job_ref_from(nil, nil, location: location),
|
336
|
-
configuration: config
|
337
|
-
)
|
409
|
+
job_object = API::Job.new job_reference: job_ref_from(nil, nil, location: location), configuration: config
|
338
410
|
# Jobs have generated id, so this operation is considered idempotent
|
339
411
|
execute backoff: true do
|
340
412
|
service.insert_job @project, job_object
|
@@ -347,18 +419,25 @@ module Google
|
|
347
419
|
end
|
348
420
|
end
|
349
421
|
|
422
|
+
##
|
423
|
+
# Deletes the job specified by jobId and location (required).
|
424
|
+
def delete_job job_id, location: nil
|
425
|
+
execute do
|
426
|
+
service.delete_job @project, job_id, location: location
|
427
|
+
end
|
428
|
+
end
|
429
|
+
|
350
430
|
##
|
351
431
|
# Returns the query data for the job
|
352
|
-
def job_query_results job_id,
|
432
|
+
def job_query_results job_id, location: nil, max: nil, token: nil, start: nil, timeout: nil
|
353
433
|
# The get operation is considered idempotent
|
354
434
|
execute backoff: true do
|
355
|
-
service.get_job_query_results
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
timeout_ms: options.delete(:timeout)
|
435
|
+
service.get_job_query_results @project, job_id,
|
436
|
+
location: location,
|
437
|
+
max_results: max,
|
438
|
+
page_token: token,
|
439
|
+
start_index: start,
|
440
|
+
timeout_ms: timeout
|
362
441
|
end
|
363
442
|
end
|
364
443
|
|
@@ -382,10 +461,7 @@ module Google
|
|
382
461
|
|
383
462
|
def load_table_file file, load_job_gapi
|
384
463
|
execute backoff: true do
|
385
|
-
service.insert_job
|
386
|
-
@project,
|
387
|
-
load_job_gapi,
|
388
|
-
upload_source: file, content_type: mime_type_for(file)
|
464
|
+
service.insert_job @project, load_job_gapi, upload_source: file, content_type: mime_type_for(file)
|
389
465
|
end
|
390
466
|
end
|
391
467
|
|
@@ -409,34 +485,42 @@ module Google
|
|
409
485
|
def self.table_ref_from_s str, default_ref: {}
|
410
486
|
str = str.to_s
|
411
487
|
m = /\A(((?<prj>\S*)(:|\.))?(?<dts>\S*)\.)?(?<tbl>\S*)\z/.match str
|
412
|
-
unless m
|
413
|
-
raise ArgumentError, "unable to identify table from #{str.inspect}"
|
414
|
-
end
|
488
|
+
raise ArgumentError, "unable to identify table from #{str.inspect}" unless m
|
415
489
|
str_table_ref_hash = {
|
416
490
|
project_id: m["prj"],
|
417
491
|
dataset_id: m["dts"],
|
418
492
|
table_id: m["tbl"]
|
419
493
|
}.delete_if { |_, v| v.nil? }
|
420
494
|
str_table_ref_hash = default_ref.to_h.merge str_table_ref_hash
|
421
|
-
ref = Google::Apis::BigqueryV2::TableReference.new
|
495
|
+
ref = Google::Apis::BigqueryV2::TableReference.new(**str_table_ref_hash)
|
422
496
|
validate_table_ref ref
|
423
497
|
ref
|
424
498
|
end
|
425
499
|
|
500
|
+
##
|
501
|
+
# Converts a hash to a Google::Apis::BigqueryV2::DatasetAccessEntry oject.
|
502
|
+
#
|
503
|
+
# @param [Hash<String,String>] dataset_hash Hash for a DatasetAccessEntry.
|
504
|
+
#
|
505
|
+
def self.dataset_access_entry_from_hash dataset_hash
|
506
|
+
params = {
|
507
|
+
dataset: Google::Apis::BigqueryV2::DatasetReference.new(**dataset_hash),
|
508
|
+
target_types: dataset_hash[:target_types]
|
509
|
+
}.delete_if { |_, v| v.nil? }
|
510
|
+
Google::Apis::BigqueryV2::DatasetAccessEntry.new(**params)
|
511
|
+
end
|
512
|
+
|
426
513
|
def self.validate_table_ref table_ref
|
427
|
-
|
428
|
-
if table_ref.send(f).nil?
|
429
|
-
raise ArgumentError, "TableReference is missing #{f}"
|
430
|
-
end
|
514
|
+
[:project_id, :dataset_id, :table_id].each do |f|
|
515
|
+
raise ArgumentError, "TableReference is missing #{f}" if table_ref.send(f).nil?
|
431
516
|
end
|
432
517
|
end
|
433
518
|
|
434
519
|
##
|
435
520
|
# Lists all projects to which you have been granted any project role.
|
436
|
-
def list_projects
|
521
|
+
def list_projects max: nil, token: nil
|
437
522
|
execute backoff: true do
|
438
|
-
service.list_projects max_results:
|
439
|
-
page_token: options[:token]
|
523
|
+
service.list_projects max_results: max, page_token: token
|
440
524
|
end
|
441
525
|
end
|
442
526
|
|
@@ -446,10 +530,7 @@ module Google
|
|
446
530
|
def job_ref_from job_id, prefix, location: nil
|
447
531
|
prefix ||= "job_"
|
448
532
|
job_id ||= "#{prefix}#{generate_id}"
|
449
|
-
job_ref = API::JobReference.new
|
450
|
-
project_id: @project,
|
451
|
-
job_id: job_id
|
452
|
-
)
|
533
|
+
job_ref = API::JobReference.new project_id: @project, job_id: job_id
|
453
534
|
# BigQuery does not allow nil location, but missing is ok.
|
454
535
|
job_ref.location = location if location
|
455
536
|
job_ref
|
@@ -477,6 +558,11 @@ module Google
|
|
477
558
|
|
478
559
|
protected
|
479
560
|
|
561
|
+
# Creates a formatted table path.
|
562
|
+
def table_path dataset_id, table_id
|
563
|
+
"projects/#{@project}/datasets/#{dataset_id}/tables/#{table_id}"
|
564
|
+
end
|
565
|
+
|
480
566
|
# Generate a random string similar to the BigQuery service job IDs.
|
481
567
|
def generate_id
|
482
568
|
SecureRandom.urlsafe_base64 21
|
@@ -490,9 +576,9 @@ module Google
|
|
490
576
|
nil
|
491
577
|
end
|
492
578
|
|
493
|
-
def execute backoff: nil
|
579
|
+
def execute backoff: nil, &block
|
494
580
|
if backoff
|
495
|
-
Backoff.new(retries: retries).execute
|
581
|
+
Backoff.new(retries: retries).execute(&block)
|
496
582
|
else
|
497
583
|
yield
|
498
584
|
end
|
@@ -500,6 +586,14 @@ module Google
|
|
500
586
|
raise Google::Cloud::Error.from_error e
|
501
587
|
end
|
502
588
|
|
589
|
+
def table_metadata_view_type_for str
|
590
|
+
return nil if str.nil?
|
591
|
+
{ "unspecified" => "TABLE_METADATA_VIEW_UNSPECIFIED",
|
592
|
+
"basic" => "BASIC",
|
593
|
+
"storage" => "STORAGE_STATS",
|
594
|
+
"full" => "FULL" }[str.to_s.downcase]
|
595
|
+
end
|
596
|
+
|
503
597
|
class Backoff
|
504
598
|
class << self
|
505
599
|
attr_accessor :retries
|
@@ -507,7 +601,7 @@ module Google
|
|
507
601
|
attr_accessor :backoff
|
508
602
|
end
|
509
603
|
self.retries = 5
|
510
|
-
self.reasons =
|
604
|
+
self.reasons = ["rateLimitExceeded", "backendError"]
|
511
605
|
self.backoff = lambda do |retries|
|
512
606
|
# Max delay is 32 seconds
|
513
607
|
# See "Back-off Requirements" here:
|
@@ -517,31 +611,29 @@ module Google
|
|
517
611
|
sleep delay
|
518
612
|
end
|
519
613
|
|
520
|
-
def initialize
|
521
|
-
@retries = (
|
522
|
-
@reasons = (
|
523
|
-
@backoff =
|
614
|
+
def initialize retries: nil, reasons: nil, backoff: nil
|
615
|
+
@retries = (retries || Backoff.retries).to_i
|
616
|
+
@reasons = (reasons || Backoff.reasons).to_a
|
617
|
+
@backoff = backoff || Backoff.backoff
|
524
618
|
end
|
525
619
|
|
526
620
|
def execute
|
527
621
|
current_retries = 0
|
528
622
|
loop do
|
529
|
-
|
530
|
-
|
531
|
-
|
532
|
-
|
533
|
-
|
534
|
-
|
535
|
-
current_retries += 1
|
536
|
-
end
|
623
|
+
return yield
|
624
|
+
rescue Google::Apis::Error => e
|
625
|
+
raise e unless retry? e.body, current_retries
|
626
|
+
|
627
|
+
@backoff.call current_retries
|
628
|
+
current_retries += 1
|
537
629
|
end
|
538
630
|
end
|
539
631
|
|
540
632
|
protected
|
541
633
|
|
542
|
-
def retry? result, current_retries
|
543
|
-
if current_retries < @retries
|
544
|
-
return true
|
634
|
+
def retry? result, current_retries
|
635
|
+
if current_retries < @retries && retry_error_reason?(result)
|
636
|
+
return true
|
545
637
|
end
|
546
638
|
false
|
547
639
|
end
|