google-cloud-bigquery 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/google-cloud-bigquery.rb +25 -0
- data/lib/google/cloud/bigquery.rb +61 -11
- data/lib/google/cloud/bigquery/convert.rb +1 -1
- data/lib/google/cloud/bigquery/credentials.rb +6 -6
- data/lib/google/cloud/bigquery/data.rb +6 -6
- data/lib/google/cloud/bigquery/dataset.rb +16 -15
- data/lib/google/cloud/bigquery/dataset/access.rb +38 -30
- data/lib/google/cloud/bigquery/dataset/list.rb +1 -1
- data/lib/google/cloud/bigquery/external.rb +22 -20
- data/lib/google/cloud/bigquery/insert_response.rb +0 -2
- data/lib/google/cloud/bigquery/job.rb +39 -31
- data/lib/google/cloud/bigquery/job/list.rb +1 -1
- data/lib/google/cloud/bigquery/load_job.rb +4 -4
- data/lib/google/cloud/bigquery/project.rb +7 -15
- data/lib/google/cloud/bigquery/project/list.rb +1 -1
- data/lib/google/cloud/bigquery/query_job.rb +12 -12
- data/lib/google/cloud/bigquery/schema.rb +7 -7
- data/lib/google/cloud/bigquery/schema/field.rb +12 -12
- data/lib/google/cloud/bigquery/service.rb +44 -29
- data/lib/google/cloud/bigquery/table.rb +78 -21
- data/lib/google/cloud/bigquery/table/async_inserter.rb +42 -17
- data/lib/google/cloud/bigquery/table/list.rb +1 -1
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +11 -11
@@ -55,7 +55,7 @@ module Google
|
|
55
55
|
def self.from_urls urls, format = nil
|
56
56
|
external_format = source_format_for urls, format
|
57
57
|
if external_format.nil?
|
58
|
-
|
58
|
+
raise ArgumentError, "Unable to determine external table format"
|
59
59
|
end
|
60
60
|
external_class = table_class_for external_format
|
61
61
|
external_class.new.tap do |e|
|
@@ -70,7 +70,7 @@ module Google
|
|
70
70
|
external_format = source_format_for gapi.source_uris,
|
71
71
|
gapi.source_format
|
72
72
|
if external_format.nil?
|
73
|
-
|
73
|
+
raise ArgumentError, "Unable to determine external table format"
|
74
74
|
end
|
75
75
|
external_class = table_class_for external_format
|
76
76
|
external_class.from_gapi gapi
|
@@ -79,17 +79,17 @@ module Google
|
|
79
79
|
##
|
80
80
|
# @private Determine source_format from inputs
|
81
81
|
def self.source_format_for urls, format
|
82
|
-
val = {
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
82
|
+
val = {
|
83
|
+
"csv" => "CSV", "avro" => "AVRO",
|
84
|
+
"json" => "NEWLINE_DELIMITED_JSON",
|
85
|
+
"newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
|
86
|
+
"sheets" => "GOOGLE_SHEETS",
|
87
|
+
"google_sheets" => "GOOGLE_SHEETS",
|
88
|
+
"datastore" => "DATASTORE_BACKUP",
|
89
|
+
"backup" => "DATASTORE_BACKUP",
|
90
|
+
"datastore_backup" => "DATASTORE_BACKUP",
|
91
|
+
"bigtable" => "BIGTABLE"
|
92
|
+
}[format.to_s.downcase]
|
93
93
|
return val unless val.nil?
|
94
94
|
Array(urls).each do |url|
|
95
95
|
return "CSV" if url.end_with? ".csv"
|
@@ -556,7 +556,8 @@ module Google
|
|
556
556
|
|
557
557
|
def frozen_check!
|
558
558
|
return unless frozen?
|
559
|
-
|
559
|
+
raise ArgumentError,
|
560
|
+
"Cannot modify external data source when frozen"
|
560
561
|
end
|
561
562
|
end
|
562
563
|
|
@@ -1426,7 +1427,8 @@ module Google
|
|
1426
1427
|
|
1427
1428
|
def frozen_check!
|
1428
1429
|
return unless frozen?
|
1429
|
-
|
1430
|
+
raise ArgumentError,
|
1431
|
+
"Cannot modify external data source when frozen"
|
1430
1432
|
end
|
1431
1433
|
|
1432
1434
|
##
|
@@ -1964,8 +1966,8 @@ module Google
|
|
1964
1966
|
|
1965
1967
|
def frozen_check!
|
1966
1968
|
return unless frozen?
|
1967
|
-
|
1968
|
-
|
1969
|
+
raise ArgumentError,
|
1970
|
+
"Cannot modify external data source when frozen"
|
1969
1971
|
end
|
1970
1972
|
end
|
1971
1973
|
|
@@ -2064,7 +2066,7 @@ module Google
|
|
2064
2066
|
#
|
2065
2067
|
def qualifier= new_qualifier
|
2066
2068
|
frozen_check!
|
2067
|
-
|
2069
|
+
raise ArgumentError if new_qualifier.nil?
|
2068
2070
|
|
2069
2071
|
utf8_qualifier = new_qualifier.encode Encoding::UTF_8
|
2070
2072
|
if utf8_qualifier.valid_encoding?
|
@@ -2342,8 +2344,8 @@ module Google
|
|
2342
2344
|
|
2343
2345
|
def frozen_check!
|
2344
2346
|
return unless frozen?
|
2345
|
-
|
2346
|
-
|
2347
|
+
raise ArgumentError,
|
2348
|
+
"Cannot modify external data source when frozen"
|
2347
2349
|
end
|
2348
2350
|
end
|
2349
2351
|
end
|
@@ -60,7 +60,6 @@ module Google
|
|
60
60
|
error_count.zero?
|
61
61
|
end
|
62
62
|
|
63
|
-
|
64
63
|
##
|
65
64
|
# The count of rows in the response, minus the count of errors for rows
|
66
65
|
# that were not inserted.
|
@@ -71,7 +70,6 @@ module Google
|
|
71
70
|
@rows.count - error_count
|
72
71
|
end
|
73
72
|
|
74
|
-
|
75
73
|
##
|
76
74
|
# The count of errors for rows that were not inserted.
|
77
75
|
#
|
@@ -44,7 +44,7 @@ module Google
|
|
44
44
|
# bigquery = Google::Cloud::Bigquery.new
|
45
45
|
#
|
46
46
|
# job = bigquery.query_job "SELECT COUNT(word) as count FROM " \
|
47
|
-
# "
|
47
|
+
# "`bigquery-public-data.samples.shakespeare`"
|
48
48
|
#
|
49
49
|
# job.wait_until_done!
|
50
50
|
#
|
@@ -162,7 +162,7 @@ module Google
|
|
162
162
|
#
|
163
163
|
def created_at
|
164
164
|
::Time.at(Integer(@gapi.statistics.creation_time) / 1000.0)
|
165
|
-
rescue
|
165
|
+
rescue StandardError
|
166
166
|
nil
|
167
167
|
end
|
168
168
|
|
@@ -175,7 +175,7 @@ module Google
|
|
175
175
|
#
|
176
176
|
def started_at
|
177
177
|
::Time.at(Integer(@gapi.statistics.start_time) / 1000.0)
|
178
|
-
rescue
|
178
|
+
rescue StandardError
|
179
179
|
nil
|
180
180
|
end
|
181
181
|
|
@@ -187,7 +187,7 @@ module Google
|
|
187
187
|
#
|
188
188
|
def ended_at
|
189
189
|
::Time.at(Integer(@gapi.statistics.end_time) / 1000.0)
|
190
|
-
rescue
|
190
|
+
rescue StandardError
|
191
191
|
nil
|
192
192
|
end
|
193
193
|
|
@@ -199,7 +199,7 @@ module Google
|
|
199
199
|
def configuration
|
200
200
|
JSON.parse @gapi.configuration.to_json
|
201
201
|
end
|
202
|
-
|
202
|
+
alias config configuration
|
203
203
|
|
204
204
|
##
|
205
205
|
# The statistics for the job. Returns a hash.
|
@@ -212,7 +212,7 @@ module Google
|
|
212
212
|
def statistics
|
213
213
|
JSON.parse @gapi.statistics.to_json
|
214
214
|
end
|
215
|
-
|
215
|
+
alias stats statistics
|
216
216
|
|
217
217
|
##
|
218
218
|
# The job's status. Returns a hash. The values contained in the hash are
|
@@ -236,7 +236,7 @@ module Google
|
|
236
236
|
#
|
237
237
|
# {
|
238
238
|
# "reason"=>"notFound",
|
239
|
-
# "message"=>"Not found: Table
|
239
|
+
# "message"=>"Not found: Table bigquery-public-data:samples.BAD_ID"
|
240
240
|
# }
|
241
241
|
#
|
242
242
|
def error
|
@@ -252,7 +252,7 @@ module Google
|
|
252
252
|
#
|
253
253
|
# {
|
254
254
|
# "reason"=>"notFound",
|
255
|
-
# "message"=>"Not found: Table
|
255
|
+
# "message"=>"Not found: Table bigquery-public-data:samples.BAD_ID"
|
256
256
|
# }
|
257
257
|
#
|
258
258
|
def errors
|
@@ -284,8 +284,10 @@ module Google
|
|
284
284
|
#
|
285
285
|
# bigquery = Google::Cloud::Bigquery.new
|
286
286
|
#
|
287
|
-
#
|
288
|
-
#
|
287
|
+
# query = "SELECT COUNT(word) as count FROM " \
|
288
|
+
# "`bigquery-public-data.samples.shakespeare`"
|
289
|
+
#
|
290
|
+
# job = bigquery.query_job query
|
289
291
|
#
|
290
292
|
# job.cancel
|
291
293
|
#
|
@@ -304,8 +306,10 @@ module Google
|
|
304
306
|
#
|
305
307
|
# bigquery = Google::Cloud::Bigquery.new
|
306
308
|
#
|
307
|
-
#
|
308
|
-
#
|
309
|
+
# query = "SELECT COUNT(word) as count FROM " \
|
310
|
+
# "`bigquery-public-data.samples.shakespeare`"
|
311
|
+
#
|
312
|
+
# job = bigquery.query_job query
|
309
313
|
#
|
310
314
|
# job.wait_until_done!
|
311
315
|
# job.rerun!
|
@@ -324,8 +328,10 @@ module Google
|
|
324
328
|
#
|
325
329
|
# bigquery = Google::Cloud::Bigquery.new
|
326
330
|
#
|
327
|
-
#
|
328
|
-
#
|
331
|
+
# query = "SELECT COUNT(word) as count FROM " \
|
332
|
+
# "`bigquery-public-data.samples.shakespeare`"
|
333
|
+
#
|
334
|
+
# job = bigquery.query_job query
|
329
335
|
#
|
330
336
|
# job.done?
|
331
337
|
# job.reload!
|
@@ -336,7 +342,7 @@ module Google
|
|
336
342
|
gapi = service.get_job job_id
|
337
343
|
@gapi = gapi
|
338
344
|
end
|
339
|
-
|
345
|
+
alias refresh! reload!
|
340
346
|
|
341
347
|
##
|
342
348
|
# Refreshes the job until the job is `DONE`. The delay between refreshes
|
@@ -357,7 +363,7 @@ module Google
|
|
357
363
|
#
|
358
364
|
def wait_until_done!
|
359
365
|
backoff = lambda do |retries|
|
360
|
-
delay = [retries
|
366
|
+
delay = [retries**2 + 5, 60].min # Maximum delay is 60
|
361
367
|
sleep delay
|
362
368
|
end
|
363
369
|
retries = 0
|
@@ -392,27 +398,29 @@ module Google
|
|
392
398
|
body: error_body
|
393
399
|
end
|
394
400
|
|
395
|
-
protected
|
396
|
-
|
397
|
-
##
|
398
|
-
# Raise an error unless an active connection is available.
|
399
|
-
def ensure_service!
|
400
|
-
fail "Must have active connection" unless service
|
401
|
-
end
|
402
|
-
|
403
401
|
##
|
402
|
+
# @private
|
404
403
|
# Get the subclass for a job type
|
405
404
|
def self.klass_for gapi
|
406
405
|
if gapi.configuration.copy
|
407
|
-
|
406
|
+
CopyJob
|
408
407
|
elsif gapi.configuration.extract
|
409
|
-
|
408
|
+
ExtractJob
|
410
409
|
elsif gapi.configuration.load
|
411
|
-
|
410
|
+
LoadJob
|
412
411
|
elsif gapi.configuration.query
|
413
|
-
|
412
|
+
QueryJob
|
413
|
+
else
|
414
|
+
Job
|
414
415
|
end
|
415
|
-
|
416
|
+
end
|
417
|
+
|
418
|
+
protected
|
419
|
+
|
420
|
+
##
|
421
|
+
# Raise an error unless an active connection is available.
|
422
|
+
def ensure_service!
|
423
|
+
raise "Must have active connection" unless service
|
416
424
|
end
|
417
425
|
|
418
426
|
def retrieve_table project_id, dataset_id, table_id
|
@@ -427,8 +435,8 @@ module Google
|
|
427
435
|
codes = { "accessDenied" => 403, "backendError" => 500,
|
428
436
|
"billingNotEnabled" => 403,
|
429
437
|
"billingTierLimitExceeded" => 400, "blocked" => 403,
|
430
|
-
"duplicate" => 409, "internalError" =>500,
|
431
|
-
"invalidQuery" => 400, "notFound" =>404,
|
438
|
+
"duplicate" => 409, "internalError" => 500,
|
439
|
+
"invalid" => 400, "invalidQuery" => 400, "notFound" => 404,
|
432
440
|
"notImplemented" => 501, "quotaExceeded" => 403,
|
433
441
|
"rateLimitExceeded" => 403, "resourceInUse" => 400,
|
434
442
|
"resourcesExceeded" => 400, "responseTooLarge" => 403,
|
@@ -276,7 +276,7 @@ module Google
|
|
276
276
|
#
|
277
277
|
def input_files
|
278
278
|
Integer @gapi.statistics.load.input_files
|
279
|
-
rescue
|
279
|
+
rescue StandardError
|
280
280
|
nil
|
281
281
|
end
|
282
282
|
|
@@ -287,7 +287,7 @@ module Google
|
|
287
287
|
#
|
288
288
|
def input_file_bytes
|
289
289
|
Integer @gapi.statistics.load.input_file_bytes
|
290
|
-
rescue
|
290
|
+
rescue StandardError
|
291
291
|
nil
|
292
292
|
end
|
293
293
|
|
@@ -299,7 +299,7 @@ module Google
|
|
299
299
|
#
|
300
300
|
def output_rows
|
301
301
|
Integer @gapi.statistics.load.output_rows
|
302
|
-
rescue
|
302
|
+
rescue StandardError
|
303
303
|
nil
|
304
304
|
end
|
305
305
|
|
@@ -311,7 +311,7 @@ module Google
|
|
311
311
|
#
|
312
312
|
def output_bytes
|
313
313
|
Integer @gapi.statistics.load.output_bytes
|
314
|
-
rescue
|
314
|
+
rescue StandardError
|
315
315
|
nil
|
316
316
|
end
|
317
317
|
end
|
@@ -13,7 +13,6 @@
|
|
13
13
|
# limitations under the License.
|
14
14
|
|
15
15
|
|
16
|
-
require "google/cloud/env"
|
17
16
|
require "google/cloud/errors"
|
18
17
|
require "google/cloud/bigquery/service"
|
19
18
|
require "google/cloud/bigquery/credentials"
|
@@ -83,16 +82,7 @@ module Google
|
|
83
82
|
def project_id
|
84
83
|
service.project
|
85
84
|
end
|
86
|
-
|
87
|
-
|
88
|
-
##
|
89
|
-
# @private Default project_id.
|
90
|
-
def self.default_project_id
|
91
|
-
ENV["BIGQUERY_PROJECT"] ||
|
92
|
-
ENV["GOOGLE_CLOUD_PROJECT"] ||
|
93
|
-
ENV["GCLOUD_PROJECT"] ||
|
94
|
-
Google::Cloud.env.project_id
|
95
|
-
end
|
85
|
+
alias project project_id
|
96
86
|
|
97
87
|
##
|
98
88
|
# Queries data by creating a [query
|
@@ -516,8 +506,8 @@ module Google
|
|
516
506
|
if job.failed?
|
517
507
|
begin
|
518
508
|
# raise to activate ruby exception cause handling
|
519
|
-
|
520
|
-
rescue => e
|
509
|
+
raise job.gapi_error
|
510
|
+
rescue StandardError => e
|
521
511
|
# wrap Google::Apis::Error with Google::Cloud::Error
|
522
512
|
raise Google::Cloud::Error.from_error(e)
|
523
513
|
end
|
@@ -667,7 +657,9 @@ module Google
|
|
667
657
|
|
668
658
|
new_ds = Google::Apis::BigqueryV2::Dataset.new(
|
669
659
|
dataset_reference: Google::Apis::BigqueryV2::DatasetReference.new(
|
670
|
-
project_id: project, dataset_id: dataset_id
|
660
|
+
project_id: project, dataset_id: dataset_id
|
661
|
+
)
|
662
|
+
)
|
671
663
|
|
672
664
|
# Can set location only on creation, no Dataset#location method
|
673
665
|
new_ds.update! location: location unless location.nil?
|
@@ -973,7 +965,7 @@ module Google
|
|
973
965
|
##
|
974
966
|
# Raise an error unless an active service is available.
|
975
967
|
def ensure_service!
|
976
|
-
|
968
|
+
raise "Must have active connection" unless service
|
977
969
|
end
|
978
970
|
end
|
979
971
|
end
|
@@ -36,7 +36,7 @@ module Google
|
|
36
36
|
# bigquery = Google::Cloud::Bigquery.new
|
37
37
|
#
|
38
38
|
# job = bigquery.query_job "SELECT COUNT(word) as count FROM " \
|
39
|
-
# "
|
39
|
+
# "`bigquery-public-data.samples.shakespeare`"
|
40
40
|
#
|
41
41
|
# job.wait_until_done!
|
42
42
|
#
|
@@ -113,7 +113,7 @@ module Google
|
|
113
113
|
|
114
114
|
##
|
115
115
|
# Limits the billing tier for this job. Queries that have resource usage
|
116
|
-
# beyond this tier will
|
116
|
+
# beyond this tier will raise (without incurring a charge). If
|
117
117
|
# unspecified, this will be set to your project default. For more
|
118
118
|
# information, see [High-Compute
|
119
119
|
# queries](https://cloud.google.com/bigquery/pricing#high-compute).
|
@@ -127,7 +127,7 @@ module Google
|
|
127
127
|
|
128
128
|
##
|
129
129
|
# Limits the bytes billed for this job. Queries that will have bytes
|
130
|
-
# billed beyond this limit will
|
130
|
+
# billed beyond this limit will raise (without incurring a charge). If
|
131
131
|
# `nil`, this will be set to your project default.
|
132
132
|
#
|
133
133
|
# @return [Integer, nil] The number of bytes, or `nil` for the project
|
@@ -135,7 +135,7 @@ module Google
|
|
135
135
|
#
|
136
136
|
def maximum_bytes_billed
|
137
137
|
Integer @gapi.configuration.query.maximum_bytes_billed
|
138
|
-
rescue
|
138
|
+
rescue StandardError
|
139
139
|
nil
|
140
140
|
end
|
141
141
|
|
@@ -156,7 +156,7 @@ module Google
|
|
156
156
|
#
|
157
157
|
def bytes_processed
|
158
158
|
Integer @gapi.statistics.query.total_bytes_processed
|
159
|
-
rescue
|
159
|
+
rescue StandardError
|
160
160
|
nil
|
161
161
|
end
|
162
162
|
|
@@ -171,7 +171,7 @@ module Google
|
|
171
171
|
#
|
172
172
|
# bigquery = Google::Cloud::Bigquery.new
|
173
173
|
#
|
174
|
-
# sql = "SELECT word FROM
|
174
|
+
# sql = "SELECT word FROM `bigquery-public-data.samples.shakespeare`"
|
175
175
|
# job = bigquery.query_job sql
|
176
176
|
#
|
177
177
|
# job.wait_until_done!
|
@@ -253,7 +253,7 @@ module Google
|
|
253
253
|
#
|
254
254
|
# bigquery = Google::Cloud::Bigquery.new
|
255
255
|
#
|
256
|
-
# sql = "SELECT word FROM
|
256
|
+
# sql = "SELECT word FROM `bigquery-public-data.samples.shakespeare`"
|
257
257
|
# job = bigquery.query_job sql
|
258
258
|
#
|
259
259
|
# job.wait_until_done!
|
@@ -289,7 +289,7 @@ module Google
|
|
289
289
|
#
|
290
290
|
# bigquery = Google::Cloud::Bigquery.new
|
291
291
|
#
|
292
|
-
# sql = "SELECT word FROM
|
292
|
+
# sql = "SELECT word FROM `bigquery-public-data.samples.shakespeare`"
|
293
293
|
# job = bigquery.query_job sql
|
294
294
|
#
|
295
295
|
# job.wait_until_done!
|
@@ -311,7 +311,7 @@ module Google
|
|
311
311
|
options
|
312
312
|
Data.from_gapi_json data_hash, destination_table_gapi, service
|
313
313
|
end
|
314
|
-
|
314
|
+
alias query_results data
|
315
315
|
|
316
316
|
##
|
317
317
|
# Represents a stage in the execution plan for the query.
|
@@ -347,7 +347,7 @@ module Google
|
|
347
347
|
#
|
348
348
|
# bigquery = Google::Cloud::Bigquery.new
|
349
349
|
#
|
350
|
-
# sql = "SELECT word FROM
|
350
|
+
# sql = "SELECT word FROM `bigquery-public-data.samples.shakespeare`"
|
351
351
|
# job = bigquery.query_job sql
|
352
352
|
#
|
353
353
|
# job.wait_until_done!
|
@@ -416,7 +416,7 @@ module Google
|
|
416
416
|
#
|
417
417
|
# bigquery = Google::Cloud::Bigquery.new
|
418
418
|
#
|
419
|
-
# sql = "SELECT word FROM
|
419
|
+
# sql = "SELECT word FROM `bigquery-public-data.samples.shakespeare`"
|
420
420
|
# job = bigquery.query_job sql
|
421
421
|
#
|
422
422
|
# job.wait_until_done!
|
@@ -453,7 +453,7 @@ module Google
|
|
453
453
|
return unless destination_schema.nil?
|
454
454
|
|
455
455
|
query_results_gapi = service.job_query_results job_id, max: 0
|
456
|
-
#
|
456
|
+
# raise "unable to retrieve schema" if query_results_gapi.schema.nil?
|
457
457
|
@destination_schema_gapi = query_results_gapi.schema
|
458
458
|
end
|
459
459
|
|