google-cloud-bigquery 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/google-cloud-bigquery.rb +25 -0
- data/lib/google/cloud/bigquery.rb +61 -11
- data/lib/google/cloud/bigquery/convert.rb +1 -1
- data/lib/google/cloud/bigquery/credentials.rb +6 -6
- data/lib/google/cloud/bigquery/data.rb +6 -6
- data/lib/google/cloud/bigquery/dataset.rb +16 -15
- data/lib/google/cloud/bigquery/dataset/access.rb +38 -30
- data/lib/google/cloud/bigquery/dataset/list.rb +1 -1
- data/lib/google/cloud/bigquery/external.rb +22 -20
- data/lib/google/cloud/bigquery/insert_response.rb +0 -2
- data/lib/google/cloud/bigquery/job.rb +39 -31
- data/lib/google/cloud/bigquery/job/list.rb +1 -1
- data/lib/google/cloud/bigquery/load_job.rb +4 -4
- data/lib/google/cloud/bigquery/project.rb +7 -15
- data/lib/google/cloud/bigquery/project/list.rb +1 -1
- data/lib/google/cloud/bigquery/query_job.rb +12 -12
- data/lib/google/cloud/bigquery/schema.rb +7 -7
- data/lib/google/cloud/bigquery/schema/field.rb +12 -12
- data/lib/google/cloud/bigquery/service.rb +44 -29
- data/lib/google/cloud/bigquery/table.rb +78 -21
- data/lib/google/cloud/bigquery/table/async_inserter.rb +42 -17
- data/lib/google/cloud/bigquery/table/list.rb +1 -1
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +11 -11
@@ -55,7 +55,7 @@ module Google
|
|
55
55
|
def self.from_urls urls, format = nil
|
56
56
|
external_format = source_format_for urls, format
|
57
57
|
if external_format.nil?
|
58
|
-
|
58
|
+
raise ArgumentError, "Unable to determine external table format"
|
59
59
|
end
|
60
60
|
external_class = table_class_for external_format
|
61
61
|
external_class.new.tap do |e|
|
@@ -70,7 +70,7 @@ module Google
|
|
70
70
|
external_format = source_format_for gapi.source_uris,
|
71
71
|
gapi.source_format
|
72
72
|
if external_format.nil?
|
73
|
-
|
73
|
+
raise ArgumentError, "Unable to determine external table format"
|
74
74
|
end
|
75
75
|
external_class = table_class_for external_format
|
76
76
|
external_class.from_gapi gapi
|
@@ -79,17 +79,17 @@ module Google
|
|
79
79
|
##
|
80
80
|
# @private Determine source_format from inputs
|
81
81
|
def self.source_format_for urls, format
|
82
|
-
val = {
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
82
|
+
val = {
|
83
|
+
"csv" => "CSV", "avro" => "AVRO",
|
84
|
+
"json" => "NEWLINE_DELIMITED_JSON",
|
85
|
+
"newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
|
86
|
+
"sheets" => "GOOGLE_SHEETS",
|
87
|
+
"google_sheets" => "GOOGLE_SHEETS",
|
88
|
+
"datastore" => "DATASTORE_BACKUP",
|
89
|
+
"backup" => "DATASTORE_BACKUP",
|
90
|
+
"datastore_backup" => "DATASTORE_BACKUP",
|
91
|
+
"bigtable" => "BIGTABLE"
|
92
|
+
}[format.to_s.downcase]
|
93
93
|
return val unless val.nil?
|
94
94
|
Array(urls).each do |url|
|
95
95
|
return "CSV" if url.end_with? ".csv"
|
@@ -556,7 +556,8 @@ module Google
|
|
556
556
|
|
557
557
|
def frozen_check!
|
558
558
|
return unless frozen?
|
559
|
-
|
559
|
+
raise ArgumentError,
|
560
|
+
"Cannot modify external data source when frozen"
|
560
561
|
end
|
561
562
|
end
|
562
563
|
|
@@ -1426,7 +1427,8 @@ module Google
|
|
1426
1427
|
|
1427
1428
|
def frozen_check!
|
1428
1429
|
return unless frozen?
|
1429
|
-
|
1430
|
+
raise ArgumentError,
|
1431
|
+
"Cannot modify external data source when frozen"
|
1430
1432
|
end
|
1431
1433
|
|
1432
1434
|
##
|
@@ -1964,8 +1966,8 @@ module Google
|
|
1964
1966
|
|
1965
1967
|
def frozen_check!
|
1966
1968
|
return unless frozen?
|
1967
|
-
|
1968
|
-
|
1969
|
+
raise ArgumentError,
|
1970
|
+
"Cannot modify external data source when frozen"
|
1969
1971
|
end
|
1970
1972
|
end
|
1971
1973
|
|
@@ -2064,7 +2066,7 @@ module Google
|
|
2064
2066
|
#
|
2065
2067
|
def qualifier= new_qualifier
|
2066
2068
|
frozen_check!
|
2067
|
-
|
2069
|
+
raise ArgumentError if new_qualifier.nil?
|
2068
2070
|
|
2069
2071
|
utf8_qualifier = new_qualifier.encode Encoding::UTF_8
|
2070
2072
|
if utf8_qualifier.valid_encoding?
|
@@ -2342,8 +2344,8 @@ module Google
|
|
2342
2344
|
|
2343
2345
|
def frozen_check!
|
2344
2346
|
return unless frozen?
|
2345
|
-
|
2346
|
-
|
2347
|
+
raise ArgumentError,
|
2348
|
+
"Cannot modify external data source when frozen"
|
2347
2349
|
end
|
2348
2350
|
end
|
2349
2351
|
end
|
@@ -60,7 +60,6 @@ module Google
|
|
60
60
|
error_count.zero?
|
61
61
|
end
|
62
62
|
|
63
|
-
|
64
63
|
##
|
65
64
|
# The count of rows in the response, minus the count of errors for rows
|
66
65
|
# that were not inserted.
|
@@ -71,7 +70,6 @@ module Google
|
|
71
70
|
@rows.count - error_count
|
72
71
|
end
|
73
72
|
|
74
|
-
|
75
73
|
##
|
76
74
|
# The count of errors for rows that were not inserted.
|
77
75
|
#
|
@@ -44,7 +44,7 @@ module Google
|
|
44
44
|
# bigquery = Google::Cloud::Bigquery.new
|
45
45
|
#
|
46
46
|
# job = bigquery.query_job "SELECT COUNT(word) as count FROM " \
|
47
|
-
# "
|
47
|
+
# "`bigquery-public-data.samples.shakespeare`"
|
48
48
|
#
|
49
49
|
# job.wait_until_done!
|
50
50
|
#
|
@@ -162,7 +162,7 @@ module Google
|
|
162
162
|
#
|
163
163
|
def created_at
|
164
164
|
::Time.at(Integer(@gapi.statistics.creation_time) / 1000.0)
|
165
|
-
rescue
|
165
|
+
rescue StandardError
|
166
166
|
nil
|
167
167
|
end
|
168
168
|
|
@@ -175,7 +175,7 @@ module Google
|
|
175
175
|
#
|
176
176
|
def started_at
|
177
177
|
::Time.at(Integer(@gapi.statistics.start_time) / 1000.0)
|
178
|
-
rescue
|
178
|
+
rescue StandardError
|
179
179
|
nil
|
180
180
|
end
|
181
181
|
|
@@ -187,7 +187,7 @@ module Google
|
|
187
187
|
#
|
188
188
|
def ended_at
|
189
189
|
::Time.at(Integer(@gapi.statistics.end_time) / 1000.0)
|
190
|
-
rescue
|
190
|
+
rescue StandardError
|
191
191
|
nil
|
192
192
|
end
|
193
193
|
|
@@ -199,7 +199,7 @@ module Google
|
|
199
199
|
def configuration
|
200
200
|
JSON.parse @gapi.configuration.to_json
|
201
201
|
end
|
202
|
-
|
202
|
+
alias config configuration
|
203
203
|
|
204
204
|
##
|
205
205
|
# The statistics for the job. Returns a hash.
|
@@ -212,7 +212,7 @@ module Google
|
|
212
212
|
def statistics
|
213
213
|
JSON.parse @gapi.statistics.to_json
|
214
214
|
end
|
215
|
-
|
215
|
+
alias stats statistics
|
216
216
|
|
217
217
|
##
|
218
218
|
# The job's status. Returns a hash. The values contained in the hash are
|
@@ -236,7 +236,7 @@ module Google
|
|
236
236
|
#
|
237
237
|
# {
|
238
238
|
# "reason"=>"notFound",
|
239
|
-
# "message"=>"Not found: Table
|
239
|
+
# "message"=>"Not found: Table bigquery-public-data:samples.BAD_ID"
|
240
240
|
# }
|
241
241
|
#
|
242
242
|
def error
|
@@ -252,7 +252,7 @@ module Google
|
|
252
252
|
#
|
253
253
|
# {
|
254
254
|
# "reason"=>"notFound",
|
255
|
-
# "message"=>"Not found: Table
|
255
|
+
# "message"=>"Not found: Table bigquery-public-data:samples.BAD_ID"
|
256
256
|
# }
|
257
257
|
#
|
258
258
|
def errors
|
@@ -284,8 +284,10 @@ module Google
|
|
284
284
|
#
|
285
285
|
# bigquery = Google::Cloud::Bigquery.new
|
286
286
|
#
|
287
|
-
#
|
288
|
-
#
|
287
|
+
# query = "SELECT COUNT(word) as count FROM " \
|
288
|
+
# "`bigquery-public-data.samples.shakespeare`"
|
289
|
+
#
|
290
|
+
# job = bigquery.query_job query
|
289
291
|
#
|
290
292
|
# job.cancel
|
291
293
|
#
|
@@ -304,8 +306,10 @@ module Google
|
|
304
306
|
#
|
305
307
|
# bigquery = Google::Cloud::Bigquery.new
|
306
308
|
#
|
307
|
-
#
|
308
|
-
#
|
309
|
+
# query = "SELECT COUNT(word) as count FROM " \
|
310
|
+
# "`bigquery-public-data.samples.shakespeare`"
|
311
|
+
#
|
312
|
+
# job = bigquery.query_job query
|
309
313
|
#
|
310
314
|
# job.wait_until_done!
|
311
315
|
# job.rerun!
|
@@ -324,8 +328,10 @@ module Google
|
|
324
328
|
#
|
325
329
|
# bigquery = Google::Cloud::Bigquery.new
|
326
330
|
#
|
327
|
-
#
|
328
|
-
#
|
331
|
+
# query = "SELECT COUNT(word) as count FROM " \
|
332
|
+
# "`bigquery-public-data.samples.shakespeare`"
|
333
|
+
#
|
334
|
+
# job = bigquery.query_job query
|
329
335
|
#
|
330
336
|
# job.done?
|
331
337
|
# job.reload!
|
@@ -336,7 +342,7 @@ module Google
|
|
336
342
|
gapi = service.get_job job_id
|
337
343
|
@gapi = gapi
|
338
344
|
end
|
339
|
-
|
345
|
+
alias refresh! reload!
|
340
346
|
|
341
347
|
##
|
342
348
|
# Refreshes the job until the job is `DONE`. The delay between refreshes
|
@@ -357,7 +363,7 @@ module Google
|
|
357
363
|
#
|
358
364
|
def wait_until_done!
|
359
365
|
backoff = lambda do |retries|
|
360
|
-
delay = [retries
|
366
|
+
delay = [retries**2 + 5, 60].min # Maximum delay is 60
|
361
367
|
sleep delay
|
362
368
|
end
|
363
369
|
retries = 0
|
@@ -392,27 +398,29 @@ module Google
|
|
392
398
|
body: error_body
|
393
399
|
end
|
394
400
|
|
395
|
-
protected
|
396
|
-
|
397
|
-
##
|
398
|
-
# Raise an error unless an active connection is available.
|
399
|
-
def ensure_service!
|
400
|
-
fail "Must have active connection" unless service
|
401
|
-
end
|
402
|
-
|
403
401
|
##
|
402
|
+
# @private
|
404
403
|
# Get the subclass for a job type
|
405
404
|
def self.klass_for gapi
|
406
405
|
if gapi.configuration.copy
|
407
|
-
|
406
|
+
CopyJob
|
408
407
|
elsif gapi.configuration.extract
|
409
|
-
|
408
|
+
ExtractJob
|
410
409
|
elsif gapi.configuration.load
|
411
|
-
|
410
|
+
LoadJob
|
412
411
|
elsif gapi.configuration.query
|
413
|
-
|
412
|
+
QueryJob
|
413
|
+
else
|
414
|
+
Job
|
414
415
|
end
|
415
|
-
|
416
|
+
end
|
417
|
+
|
418
|
+
protected
|
419
|
+
|
420
|
+
##
|
421
|
+
# Raise an error unless an active connection is available.
|
422
|
+
def ensure_service!
|
423
|
+
raise "Must have active connection" unless service
|
416
424
|
end
|
417
425
|
|
418
426
|
def retrieve_table project_id, dataset_id, table_id
|
@@ -427,8 +435,8 @@ module Google
|
|
427
435
|
codes = { "accessDenied" => 403, "backendError" => 500,
|
428
436
|
"billingNotEnabled" => 403,
|
429
437
|
"billingTierLimitExceeded" => 400, "blocked" => 403,
|
430
|
-
"duplicate" => 409, "internalError" =>500,
|
431
|
-
"invalidQuery" => 400, "notFound" =>404,
|
438
|
+
"duplicate" => 409, "internalError" => 500,
|
439
|
+
"invalid" => 400, "invalidQuery" => 400, "notFound" => 404,
|
432
440
|
"notImplemented" => 501, "quotaExceeded" => 403,
|
433
441
|
"rateLimitExceeded" => 403, "resourceInUse" => 400,
|
434
442
|
"resourcesExceeded" => 400, "responseTooLarge" => 403,
|
@@ -276,7 +276,7 @@ module Google
|
|
276
276
|
#
|
277
277
|
def input_files
|
278
278
|
Integer @gapi.statistics.load.input_files
|
279
|
-
rescue
|
279
|
+
rescue StandardError
|
280
280
|
nil
|
281
281
|
end
|
282
282
|
|
@@ -287,7 +287,7 @@ module Google
|
|
287
287
|
#
|
288
288
|
def input_file_bytes
|
289
289
|
Integer @gapi.statistics.load.input_file_bytes
|
290
|
-
rescue
|
290
|
+
rescue StandardError
|
291
291
|
nil
|
292
292
|
end
|
293
293
|
|
@@ -299,7 +299,7 @@ module Google
|
|
299
299
|
#
|
300
300
|
def output_rows
|
301
301
|
Integer @gapi.statistics.load.output_rows
|
302
|
-
rescue
|
302
|
+
rescue StandardError
|
303
303
|
nil
|
304
304
|
end
|
305
305
|
|
@@ -311,7 +311,7 @@ module Google
|
|
311
311
|
#
|
312
312
|
def output_bytes
|
313
313
|
Integer @gapi.statistics.load.output_bytes
|
314
|
-
rescue
|
314
|
+
rescue StandardError
|
315
315
|
nil
|
316
316
|
end
|
317
317
|
end
|
@@ -13,7 +13,6 @@
|
|
13
13
|
# limitations under the License.
|
14
14
|
|
15
15
|
|
16
|
-
require "google/cloud/env"
|
17
16
|
require "google/cloud/errors"
|
18
17
|
require "google/cloud/bigquery/service"
|
19
18
|
require "google/cloud/bigquery/credentials"
|
@@ -83,16 +82,7 @@ module Google
|
|
83
82
|
def project_id
|
84
83
|
service.project
|
85
84
|
end
|
86
|
-
|
87
|
-
|
88
|
-
##
|
89
|
-
# @private Default project_id.
|
90
|
-
def self.default_project_id
|
91
|
-
ENV["BIGQUERY_PROJECT"] ||
|
92
|
-
ENV["GOOGLE_CLOUD_PROJECT"] ||
|
93
|
-
ENV["GCLOUD_PROJECT"] ||
|
94
|
-
Google::Cloud.env.project_id
|
95
|
-
end
|
85
|
+
alias project project_id
|
96
86
|
|
97
87
|
##
|
98
88
|
# Queries data by creating a [query
|
@@ -516,8 +506,8 @@ module Google
|
|
516
506
|
if job.failed?
|
517
507
|
begin
|
518
508
|
# raise to activate ruby exception cause handling
|
519
|
-
|
520
|
-
rescue => e
|
509
|
+
raise job.gapi_error
|
510
|
+
rescue StandardError => e
|
521
511
|
# wrap Google::Apis::Error with Google::Cloud::Error
|
522
512
|
raise Google::Cloud::Error.from_error(e)
|
523
513
|
end
|
@@ -667,7 +657,9 @@ module Google
|
|
667
657
|
|
668
658
|
new_ds = Google::Apis::BigqueryV2::Dataset.new(
|
669
659
|
dataset_reference: Google::Apis::BigqueryV2::DatasetReference.new(
|
670
|
-
project_id: project, dataset_id: dataset_id
|
660
|
+
project_id: project, dataset_id: dataset_id
|
661
|
+
)
|
662
|
+
)
|
671
663
|
|
672
664
|
# Can set location only on creation, no Dataset#location method
|
673
665
|
new_ds.update! location: location unless location.nil?
|
@@ -973,7 +965,7 @@ module Google
|
|
973
965
|
##
|
974
966
|
# Raise an error unless an active service is available.
|
975
967
|
def ensure_service!
|
976
|
-
|
968
|
+
raise "Must have active connection" unless service
|
977
969
|
end
|
978
970
|
end
|
979
971
|
end
|
@@ -36,7 +36,7 @@ module Google
|
|
36
36
|
# bigquery = Google::Cloud::Bigquery.new
|
37
37
|
#
|
38
38
|
# job = bigquery.query_job "SELECT COUNT(word) as count FROM " \
|
39
|
-
# "
|
39
|
+
# "`bigquery-public-data.samples.shakespeare`"
|
40
40
|
#
|
41
41
|
# job.wait_until_done!
|
42
42
|
#
|
@@ -113,7 +113,7 @@ module Google
|
|
113
113
|
|
114
114
|
##
|
115
115
|
# Limits the billing tier for this job. Queries that have resource usage
|
116
|
-
# beyond this tier will
|
116
|
+
# beyond this tier will raise (without incurring a charge). If
|
117
117
|
# unspecified, this will be set to your project default. For more
|
118
118
|
# information, see [High-Compute
|
119
119
|
# queries](https://cloud.google.com/bigquery/pricing#high-compute).
|
@@ -127,7 +127,7 @@ module Google
|
|
127
127
|
|
128
128
|
##
|
129
129
|
# Limits the bytes billed for this job. Queries that will have bytes
|
130
|
-
# billed beyond this limit will
|
130
|
+
# billed beyond this limit will raise (without incurring a charge). If
|
131
131
|
# `nil`, this will be set to your project default.
|
132
132
|
#
|
133
133
|
# @return [Integer, nil] The number of bytes, or `nil` for the project
|
@@ -135,7 +135,7 @@ module Google
|
|
135
135
|
#
|
136
136
|
def maximum_bytes_billed
|
137
137
|
Integer @gapi.configuration.query.maximum_bytes_billed
|
138
|
-
rescue
|
138
|
+
rescue StandardError
|
139
139
|
nil
|
140
140
|
end
|
141
141
|
|
@@ -156,7 +156,7 @@ module Google
|
|
156
156
|
#
|
157
157
|
def bytes_processed
|
158
158
|
Integer @gapi.statistics.query.total_bytes_processed
|
159
|
-
rescue
|
159
|
+
rescue StandardError
|
160
160
|
nil
|
161
161
|
end
|
162
162
|
|
@@ -171,7 +171,7 @@ module Google
|
|
171
171
|
#
|
172
172
|
# bigquery = Google::Cloud::Bigquery.new
|
173
173
|
#
|
174
|
-
# sql = "SELECT word FROM
|
174
|
+
# sql = "SELECT word FROM `bigquery-public-data.samples.shakespeare`"
|
175
175
|
# job = bigquery.query_job sql
|
176
176
|
#
|
177
177
|
# job.wait_until_done!
|
@@ -253,7 +253,7 @@ module Google
|
|
253
253
|
#
|
254
254
|
# bigquery = Google::Cloud::Bigquery.new
|
255
255
|
#
|
256
|
-
# sql = "SELECT word FROM
|
256
|
+
# sql = "SELECT word FROM `bigquery-public-data.samples.shakespeare`"
|
257
257
|
# job = bigquery.query_job sql
|
258
258
|
#
|
259
259
|
# job.wait_until_done!
|
@@ -289,7 +289,7 @@ module Google
|
|
289
289
|
#
|
290
290
|
# bigquery = Google::Cloud::Bigquery.new
|
291
291
|
#
|
292
|
-
# sql = "SELECT word FROM
|
292
|
+
# sql = "SELECT word FROM `bigquery-public-data.samples.shakespeare`"
|
293
293
|
# job = bigquery.query_job sql
|
294
294
|
#
|
295
295
|
# job.wait_until_done!
|
@@ -311,7 +311,7 @@ module Google
|
|
311
311
|
options
|
312
312
|
Data.from_gapi_json data_hash, destination_table_gapi, service
|
313
313
|
end
|
314
|
-
|
314
|
+
alias query_results data
|
315
315
|
|
316
316
|
##
|
317
317
|
# Represents a stage in the execution plan for the query.
|
@@ -347,7 +347,7 @@ module Google
|
|
347
347
|
#
|
348
348
|
# bigquery = Google::Cloud::Bigquery.new
|
349
349
|
#
|
350
|
-
# sql = "SELECT word FROM
|
350
|
+
# sql = "SELECT word FROM `bigquery-public-data.samples.shakespeare`"
|
351
351
|
# job = bigquery.query_job sql
|
352
352
|
#
|
353
353
|
# job.wait_until_done!
|
@@ -416,7 +416,7 @@ module Google
|
|
416
416
|
#
|
417
417
|
# bigquery = Google::Cloud::Bigquery.new
|
418
418
|
#
|
419
|
-
# sql = "SELECT word FROM
|
419
|
+
# sql = "SELECT word FROM `bigquery-public-data.samples.shakespeare`"
|
420
420
|
# job = bigquery.query_job sql
|
421
421
|
#
|
422
422
|
# job.wait_until_done!
|
@@ -453,7 +453,7 @@ module Google
|
|
453
453
|
return unless destination_schema.nil?
|
454
454
|
|
455
455
|
query_results_gapi = service.job_query_results job_id, max: 0
|
456
|
-
#
|
456
|
+
# raise "unable to retrieve schema" if query_results_gapi.schema.nil?
|
457
457
|
@destination_schema_gapi = query_results_gapi.schema
|
458
458
|
end
|
459
459
|
|