google-cloud-bigquery 1.1.0 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +27 -19
- data/lib/google/cloud/bigquery/convert.rb +106 -13
- data/lib/google/cloud/bigquery/copy_job.rb +168 -0
- data/lib/google/cloud/bigquery/dataset.rb +313 -112
- data/lib/google/cloud/bigquery/encryption_configuration.rb +124 -0
- data/lib/google/cloud/bigquery/extract_job.rb +149 -0
- data/lib/google/cloud/bigquery/job.rb +14 -3
- data/lib/google/cloud/bigquery/load_job.rb +770 -0
- data/lib/google/cloud/bigquery/project.rb +118 -20
- data/lib/google/cloud/bigquery/query_job.rb +403 -2
- data/lib/google/cloud/bigquery/schema/field.rb +22 -22
- data/lib/google/cloud/bigquery/service.rb +45 -328
- data/lib/google/cloud/bigquery/table.rb +375 -95
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +3 -2
@@ -108,6 +108,9 @@ module Google
|
|
108
108
|
# See [Data Types](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types)
|
109
109
|
# for an overview of each BigQuery data type, including allowed values.
|
110
110
|
#
|
111
|
+
# The geographic location for the job ("US", "EU", etc.) can be set via
|
112
|
+
# {QueryJob::Updater#location=} in a block passed to this method.
|
113
|
+
#
|
111
114
|
# @param [String] query A query string, following the BigQuery [query
|
112
115
|
# syntax](https://cloud.google.com/bigquery/query-reference), of the
|
113
116
|
# query to execute. Example: "SELECT count(f1) FROM
|
@@ -178,11 +181,6 @@ module Google
|
|
178
181
|
# Flattens all nested and repeated fields in the query results. The
|
179
182
|
# default value is `true`. `large_results` parameter must be `true` if
|
180
183
|
# this is set to `false`.
|
181
|
-
# @param [Integer] maximum_billing_tier Limits the billing tier for this
|
182
|
-
# job. Queries that have resource usage beyond this tier will fail
|
183
|
-
# (without incurring a charge). Optional. If unspecified, this will be
|
184
|
-
# set to your project default. For more information, see [High-Compute
|
185
|
-
# queries](https://cloud.google.com/bigquery/pricing#high-compute).
|
186
184
|
# @param [Integer] maximum_bytes_billed Limits the bytes billed for this
|
187
185
|
# job. Queries that will have bytes billed beyond this limit will fail
|
188
186
|
# (without incurring a charge). Optional. If unspecified, this will be
|
@@ -219,6 +217,11 @@ module Google
|
|
219
217
|
# inline code resource is equivalent to providing a URI for a file
|
220
218
|
# containing the same code. See [User-Defined
|
221
219
|
# Functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/user-defined-functions).
|
220
|
+
# @param [Integer] maximum_billing_tier Deprecated: Change the billing
|
221
|
+
# tier to allow high-compute queries.
|
222
|
+
# @yield [job] a job configuration object
|
223
|
+
# @yieldparam [Google::Cloud::Bigquery::QueryJob::Updater] job a job
|
224
|
+
# configuration object for setting query options.
|
222
225
|
#
|
223
226
|
# @return [Google::Cloud::Bigquery::QueryJob]
|
224
227
|
#
|
@@ -243,7 +246,7 @@ module Google
|
|
243
246
|
# bigquery = Google::Cloud::Bigquery.new
|
244
247
|
#
|
245
248
|
# job = bigquery.query_job "SELECT name FROM " \
|
246
|
-
# "[my_project:my_dataset.my_table]",
|
249
|
+
# " [my_project:my_dataset.my_table]",
|
247
250
|
# legacy_sql: true
|
248
251
|
#
|
249
252
|
# job.wait_until_done!
|
@@ -287,7 +290,7 @@ module Google
|
|
287
290
|
# end
|
288
291
|
# end
|
289
292
|
#
|
290
|
-
# @example Query using external data source:
|
293
|
+
# @example Query using external data source, set destination:
|
291
294
|
# require "google/cloud/bigquery"
|
292
295
|
#
|
293
296
|
# bigquery = Google::Cloud::Bigquery.new
|
@@ -298,8 +301,11 @@ module Google
|
|
298
301
|
# csv.skip_leading_rows = 1
|
299
302
|
# end
|
300
303
|
#
|
301
|
-
# job = bigquery.query_job "SELECT * FROM my_ext_table"
|
302
|
-
#
|
304
|
+
# job = bigquery.query_job "SELECT * FROM my_ext_table" do |query|
|
305
|
+
# query.external = { my_ext_table: csv_table }
|
306
|
+
# dataset = bigquery.dataset "my_dataset", skip_lookup: true
|
307
|
+
# query.table = dataset.table "my_table", skip_lookup: true
|
308
|
+
# end
|
303
309
|
#
|
304
310
|
# job.wait_until_done!
|
305
311
|
# if !job.failed?
|
@@ -319,13 +325,18 @@ module Google
|
|
319
325
|
options = { priority: priority, cache: cache, table: table,
|
320
326
|
create: create, write: write,
|
321
327
|
large_results: large_results, flatten: flatten,
|
322
|
-
dataset: dataset, project: project,
|
328
|
+
dataset: dataset, project: (project || self.project),
|
323
329
|
legacy_sql: legacy_sql, standard_sql: standard_sql,
|
324
330
|
maximum_billing_tier: maximum_billing_tier,
|
325
331
|
maximum_bytes_billed: maximum_bytes_billed,
|
326
|
-
|
327
|
-
|
328
|
-
|
332
|
+
external: external, job_id: job_id, prefix: prefix,
|
333
|
+
labels: labels, udfs: udfs, params: params }
|
334
|
+
|
335
|
+
updater = QueryJob::Updater.from_options service, query, options
|
336
|
+
|
337
|
+
yield updater if block_given?
|
338
|
+
|
339
|
+
gapi = service.query_job updater.to_gapi
|
329
340
|
Job.from_gapi gapi, service
|
330
341
|
end
|
331
342
|
|
@@ -355,6 +366,9 @@ module Google
|
|
355
366
|
# See [Data Types](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types)
|
356
367
|
# for an overview of each BigQuery data type, including allowed values.
|
357
368
|
#
|
369
|
+
# The geographic location for the job ("US", "EU", etc.) can be set via
|
370
|
+
# {QueryJob::Updater#location=} in a block passed to this method.
|
371
|
+
#
|
358
372
|
# @see https://cloud.google.com/bigquery/querying-data Querying Data
|
359
373
|
#
|
360
374
|
# @param [String] query A query string, following the BigQuery [query
|
@@ -409,6 +423,9 @@ module Google
|
|
409
423
|
# When set to false, the values of `large_results` and `flatten` are
|
410
424
|
# ignored; the query will be run as if `large_results` is true and
|
411
425
|
# `flatten` is false. Optional. The default value is false.
|
426
|
+
# @yield [job] a job configuration object
|
427
|
+
# @yieldparam [Google::Cloud::Bigquery::QueryJob::Updater] job a job
|
428
|
+
# configuration object for setting additional options for the query.
|
412
429
|
#
|
413
430
|
# @return [Google::Cloud::Bigquery::Data]
|
414
431
|
#
|
@@ -475,7 +492,7 @@ module Google
|
|
475
492
|
# puts row[:name]
|
476
493
|
# end
|
477
494
|
#
|
478
|
-
# @example Query using external data source:
|
495
|
+
# @example Query using external data source, set destination:
|
479
496
|
# require "google/cloud/bigquery"
|
480
497
|
#
|
481
498
|
# bigquery = Google::Cloud::Bigquery.new
|
@@ -486,8 +503,11 @@ module Google
|
|
486
503
|
# csv.skip_leading_rows = 1
|
487
504
|
# end
|
488
505
|
#
|
489
|
-
# data = bigquery.query "SELECT * FROM my_ext_table"
|
490
|
-
#
|
506
|
+
# data = bigquery.query "SELECT * FROM my_ext_table" do |query|
|
507
|
+
# query.external = { my_ext_table: csv_table }
|
508
|
+
# dataset = bigquery.dataset "my_dataset", skip_lookup: true
|
509
|
+
# query.table = dataset.table "my_table", skip_lookup: true
|
510
|
+
# end
|
491
511
|
#
|
492
512
|
# data.each do |row|
|
493
513
|
# puts row[:name]
|
@@ -496,11 +516,16 @@ module Google
|
|
496
516
|
def query query, params: nil, external: nil, max: nil, cache: true,
|
497
517
|
dataset: nil, project: nil, standard_sql: nil, legacy_sql: nil
|
498
518
|
ensure_service!
|
499
|
-
options = { cache: cache, dataset: dataset,
|
519
|
+
options = { priority: "INTERACTIVE", cache: cache, dataset: dataset,
|
520
|
+
project: project || self.project,
|
500
521
|
legacy_sql: legacy_sql, standard_sql: standard_sql,
|
501
522
|
params: params, external: external }
|
523
|
+
updater = QueryJob::Updater.from_options service, query, options
|
524
|
+
|
525
|
+
yield updater if block_given?
|
502
526
|
|
503
|
-
|
527
|
+
gapi = service.query_job updater.to_gapi
|
528
|
+
job = Job.from_gapi gapi, service
|
504
529
|
job.wait_until_done!
|
505
530
|
|
506
531
|
if job.failed?
|
@@ -734,6 +759,8 @@ module Google
|
|
734
759
|
# Retrieves an existing job by ID.
|
735
760
|
#
|
736
761
|
# @param [String] job_id The ID of a job.
|
762
|
+
# @param [String] location The geographic location where the job was
|
763
|
+
# created. Required except for US and EU.
|
737
764
|
#
|
738
765
|
# @return [Google::Cloud::Bigquery::Job, nil] Returns `nil` if the job
|
739
766
|
# does not exist.
|
@@ -745,9 +772,9 @@ module Google
|
|
745
772
|
#
|
746
773
|
# job = bigquery.job "my_job"
|
747
774
|
#
|
748
|
-
def job job_id
|
775
|
+
def job job_id, location: nil
|
749
776
|
ensure_service!
|
750
|
-
gapi = service.get_job job_id
|
777
|
+
gapi = service.get_job job_id, location: location
|
751
778
|
Job.from_gapi gapi, service
|
752
779
|
rescue Google::Cloud::NotFoundError
|
753
780
|
nil
|
@@ -941,6 +968,77 @@ module Google
|
|
941
968
|
s
|
942
969
|
end
|
943
970
|
|
971
|
+
##
|
972
|
+
# Creates a new Bigquery::EncryptionConfiguration instance.
|
973
|
+
#
|
974
|
+
# This method does not execute an API call. Use the encryption
|
975
|
+
# configuration to encrypt a table when creating one via
|
976
|
+
# Bigquery::Dataset#create_table, Bigquery::Dataset#load,
|
977
|
+
# Bigquery::Table#copy, or Bigquery::Project#query.
|
978
|
+
#
|
979
|
+
# @param [String] kms_key Name of the Cloud KMS encryption key that
|
980
|
+
# will be used to protect the destination BigQuery table. The BigQuery
|
981
|
+
# Service Account associated with your project requires access to this
|
982
|
+
# encryption key.
|
983
|
+
#
|
984
|
+
# @example Encrypt a new table
|
985
|
+
# require "google/cloud/bigquery"
|
986
|
+
#
|
987
|
+
# bigquery = Google::Cloud::Bigquery.new
|
988
|
+
# dataset = bigquery.dataset "my_dataset"
|
989
|
+
#
|
990
|
+
# key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
|
991
|
+
# encrypt_config = bigquery.encryption kms_key: key_name
|
992
|
+
#
|
993
|
+
# table = dataset.create_table "my_table" do |updater|
|
994
|
+
# updater.encryption = encrypt_config
|
995
|
+
# end
|
996
|
+
#
|
997
|
+
# @example Encrypt a load destination table
|
998
|
+
# require "google/cloud/bigquery"
|
999
|
+
#
|
1000
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1001
|
+
# dataset = bigquery.dataset "my_dataset"
|
1002
|
+
#
|
1003
|
+
# key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
|
1004
|
+
# encrypt_config = bigquery.encryption kms_key: key_name
|
1005
|
+
# job = dataset.load_job "my_table", "gs://abc/file" do |job|
|
1006
|
+
# job.encryption = encrypt_config
|
1007
|
+
# end
|
1008
|
+
#
|
1009
|
+
# @example Encrypt a copy destination table
|
1010
|
+
# require "google/cloud/bigquery"
|
1011
|
+
#
|
1012
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1013
|
+
# dataset = bigquery.dataset "my_dataset"
|
1014
|
+
# table = dataset.table "my_table"
|
1015
|
+
#
|
1016
|
+
# key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
|
1017
|
+
# encrypt_config = bigquery.encryption kms_key: key_name
|
1018
|
+
# job = table.copy_job "my_dataset.new_table" do |job|
|
1019
|
+
# job.encryption = encrypt_config
|
1020
|
+
# end
|
1021
|
+
#
|
1022
|
+
# @example Encrypt a query destination table
|
1023
|
+
# require "google/cloud/bigquery"
|
1024
|
+
#
|
1025
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1026
|
+
# dataset = bigquery.dataset "my_dataset"
|
1027
|
+
#
|
1028
|
+
# key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
|
1029
|
+
# encrypt_config = bigquery.encryption kms_key: key_name
|
1030
|
+
# job = bigquery.query_job "SELECT 1;" do |query|
|
1031
|
+
# query.table = dataset.table "my_table", skip_lookup: true
|
1032
|
+
# query.encryption = encrypt_config
|
1033
|
+
# end
|
1034
|
+
#
|
1035
|
+
# @return [Google::Cloud::Bigquery::EncryptionConfiguration]
|
1036
|
+
def encryption kms_key: nil
|
1037
|
+
encrypt_config = Bigquery::EncryptionConfiguration.new
|
1038
|
+
encrypt_config.kms_key = kms_key unless kms_key.nil?
|
1039
|
+
encrypt_config
|
1040
|
+
end
|
1041
|
+
|
944
1042
|
##
|
945
1043
|
# @private New Project from a Google API Client object, using the
|
946
1044
|
# same Credentials as this project.
|
@@ -15,6 +15,8 @@
|
|
15
15
|
|
16
16
|
require "google/cloud/bigquery/service"
|
17
17
|
require "google/cloud/bigquery/data"
|
18
|
+
require "google/cloud/bigquery/encryption_configuration"
|
19
|
+
require "google/apis/bigquery_v2"
|
18
20
|
|
19
21
|
module Google
|
20
22
|
module Cloud
|
@@ -244,6 +246,19 @@ module Google
|
|
244
246
|
end
|
245
247
|
end
|
246
248
|
|
249
|
+
##
|
250
|
+
# The encryption configuration of the destination table.
|
251
|
+
#
|
252
|
+
# @return [Google::Cloud::BigQuery::EncryptionConfiguration] Custom
|
253
|
+
# encryption configuration (e.g., Cloud KMS keys).
|
254
|
+
#
|
255
|
+
# @!group Attributes
|
256
|
+
def encryption
|
257
|
+
EncryptionConfiguration.from_gapi(
|
258
|
+
@gapi.configuration.query.destination_encryption_configuration
|
259
|
+
)
|
260
|
+
end
|
261
|
+
|
247
262
|
##
|
248
263
|
# Refreshes the job until the job is `DONE`.
|
249
264
|
# The delay between refreshes will incrementally increase.
|
@@ -264,7 +279,8 @@ module Google
|
|
264
279
|
|
265
280
|
ensure_service!
|
266
281
|
loop do
|
267
|
-
query_results_gapi = service.job_query_results
|
282
|
+
query_results_gapi = service.job_query_results \
|
283
|
+
job_id, location: location, max: 0
|
268
284
|
if query_results_gapi.job_complete
|
269
285
|
@destination_schema_gapi = query_results_gapi.schema
|
270
286
|
break
|
@@ -313,6 +329,390 @@ module Google
|
|
313
329
|
end
|
314
330
|
alias query_results data
|
315
331
|
|
332
|
+
##
|
333
|
+
# Yielded to a block to accumulate changes for a patch request.
|
334
|
+
class Updater < QueryJob
|
335
|
+
##
|
336
|
+
# @private Create an Updater object.
|
337
|
+
def initialize service, gapi
|
338
|
+
@service = service
|
339
|
+
@gapi = gapi
|
340
|
+
end
|
341
|
+
|
342
|
+
# rubocop:disable all
|
343
|
+
|
344
|
+
##
|
345
|
+
# @private Create an Updater from an options hash.
|
346
|
+
#
|
347
|
+
# @return [Google::Cloud::Bigquery::QueryJob::Updater] A job
|
348
|
+
# configuration object for setting query options.
|
349
|
+
def self.from_options service, query, options
|
350
|
+
job_ref = service.job_ref_from options[:job_id], options[:prefix]
|
351
|
+
dataset_config = service.dataset_ref_from options[:dataset],
|
352
|
+
options[:project]
|
353
|
+
req = Google::Apis::BigqueryV2::Job.new(
|
354
|
+
job_reference: job_ref,
|
355
|
+
configuration: Google::Apis::BigqueryV2::JobConfiguration.new(
|
356
|
+
query: Google::Apis::BigqueryV2::JobConfigurationQuery.new(
|
357
|
+
query: query,
|
358
|
+
default_dataset: dataset_config,
|
359
|
+
maximum_billing_tier: options[:maximum_billing_tier]
|
360
|
+
)
|
361
|
+
)
|
362
|
+
)
|
363
|
+
|
364
|
+
updater = QueryJob::Updater.new service, req
|
365
|
+
updater.params = options[:params] if options[:params]
|
366
|
+
updater.create = options[:create]
|
367
|
+
updater.write = options[:write]
|
368
|
+
updater.table = options[:table]
|
369
|
+
updater.maximum_bytes_billed = options[:maximum_bytes_billed]
|
370
|
+
updater.labels = options[:labels] if options[:labels]
|
371
|
+
updater.legacy_sql = Convert.resolve_legacy_sql(
|
372
|
+
options[:standard_sql], options[:legacy_sql])
|
373
|
+
updater.external = options[:external] if options[:external]
|
374
|
+
updater.priority = options[:priority]
|
375
|
+
updater.cache = options[:cache]
|
376
|
+
updater.large_results = options[:large_results]
|
377
|
+
updater.flatten = options[:flatten]
|
378
|
+
updater.udfs = options[:udfs]
|
379
|
+
updater
|
380
|
+
end
|
381
|
+
|
382
|
+
# rubocop:enable all
|
383
|
+
|
384
|
+
##
|
385
|
+
# Sets the geographic location where the job should run. Required
|
386
|
+
# except for US and EU.
|
387
|
+
#
|
388
|
+
# @param [String] value A geographic location, such as "US", "EU" or
|
389
|
+
# "asia-northeast1". Required except for US and EU.
|
390
|
+
#
|
391
|
+
# @example
|
392
|
+
# require "google/cloud/bigquery"
|
393
|
+
#
|
394
|
+
# bigquery = Google::Cloud::Bigquery.new
|
395
|
+
# dataset = bigquery.dataset "my_dataset"
|
396
|
+
#
|
397
|
+
# job = bigquery.query_job "SELECT 1;" do |query|
|
398
|
+
# query.table = dataset.table "my_table", skip_lookup: true
|
399
|
+
# query.location = "EU"
|
400
|
+
# end
|
401
|
+
#
|
402
|
+
# @!group Attributes
|
403
|
+
def location= value
|
404
|
+
@gapi.job_reference.location = value
|
405
|
+
end
|
406
|
+
|
407
|
+
##
|
408
|
+
# Sets the priority of the query.
|
409
|
+
#
|
410
|
+
# @param [String] value Specifies a priority for the query. Possible
|
411
|
+
# values include `INTERACTIVE` and `BATCH`.
|
412
|
+
#
|
413
|
+
# @!group Attributes
|
414
|
+
def priority= value
|
415
|
+
@gapi.configuration.query.priority = priority_value value
|
416
|
+
end
|
417
|
+
|
418
|
+
##
|
419
|
+
# Specifies to look in the query cache for results.
|
420
|
+
#
|
421
|
+
# @param [Boolean] value Whether to look for the result in the query
|
422
|
+
# cache. The query cache is a best-effort cache that will be flushed
|
423
|
+
# whenever tables in the query are modified. The default value is
|
424
|
+
# true. For more information, see [query
|
425
|
+
# caching](https://developers.google.com/bigquery/querying-data).
|
426
|
+
#
|
427
|
+
# @!group Attributes
|
428
|
+
def cache= value
|
429
|
+
@gapi.configuration.query.use_query_cache = value
|
430
|
+
end
|
431
|
+
|
432
|
+
##
|
433
|
+
# Allow large results for a legacy SQL query.
|
434
|
+
#
|
435
|
+
# @param [Boolean] value This option is specific to Legacy SQL.
|
436
|
+
# If `true`, allows the query to produce arbitrarily large result
|
437
|
+
# tables at a slight cost in performance. Requires `table` parameter
|
438
|
+
# to be set.
|
439
|
+
#
|
440
|
+
# @!group Attributes
|
441
|
+
def large_results= value
|
442
|
+
@gapi.configuration.query.allow_large_results = value
|
443
|
+
end
|
444
|
+
|
445
|
+
##
|
446
|
+
# Flatten nested and repeated fields in legacy SQL queries.
|
447
|
+
#
|
448
|
+
# @param [Boolean] value This option is specific to Legacy SQL.
|
449
|
+
# Flattens all nested and repeated fields in the query results. The
|
450
|
+
# default value is `true`. `large_results` parameter must be `true`
|
451
|
+
# if this is set to `false`.
|
452
|
+
#
|
453
|
+
# @!group Attributes
|
454
|
+
def flatten= value
|
455
|
+
@gapi.configuration.query.flatten_results = value
|
456
|
+
end
|
457
|
+
|
458
|
+
##
|
459
|
+
# Sets the default dataset of tables referenced in the query.
|
460
|
+
#
|
461
|
+
# @param [Dataset] value The default dataset to use for unqualified
|
462
|
+
# table names in the query.
|
463
|
+
#
|
464
|
+
# @!group Attributes
|
465
|
+
def dataset= value
|
466
|
+
@gapi.configuration.query.default_dataset =
|
467
|
+
@service.dataset_ref_from value
|
468
|
+
end
|
469
|
+
|
470
|
+
##
|
471
|
+
# Sets the query parameters. Standard SQL only.
|
472
|
+
#
|
473
|
+
# @param [Array, Hash] params Used to pass query arguments when the
|
474
|
+
# `query` string contains either positional (`?`) or named
|
475
|
+
# (`@myparam`) query parameters. If value passed is an array
|
476
|
+
# `["foo"]`, the query must use positional query parameters. If
|
477
|
+
# value passed is a hash `{ myparam: "foo" }`, the query must use
|
478
|
+
# named query parameters. When set, `legacy_sql` will automatically
|
479
|
+
# be set to false and `standard_sql` to true.
|
480
|
+
#
|
481
|
+
# @!group Attributes
|
482
|
+
def params= params
|
483
|
+
case params
|
484
|
+
when Array then
|
485
|
+
@gapi.configuration.query.use_legacy_sql = false
|
486
|
+
@gapi.configuration.query.parameter_mode = "POSITIONAL"
|
487
|
+
@gapi.configuration.query.query_parameters = params.map do |param|
|
488
|
+
Convert.to_query_param param
|
489
|
+
end
|
490
|
+
when Hash then
|
491
|
+
@gapi.configuration.query.use_legacy_sql = false
|
492
|
+
@gapi.configuration.query.parameter_mode = "NAMED"
|
493
|
+
@gapi.configuration.query.query_parameters =
|
494
|
+
params.map do |name, param|
|
495
|
+
Convert.to_query_param(param).tap do |named_param|
|
496
|
+
named_param.name = String name
|
497
|
+
end
|
498
|
+
end
|
499
|
+
else
|
500
|
+
raise "Query parameters must be an Array or a Hash."
|
501
|
+
end
|
502
|
+
end
|
503
|
+
|
504
|
+
##
|
505
|
+
# Sets the create disposition for creating the query results table.
|
506
|
+
#
|
507
|
+
# @param [String] value Specifies whether the job is allowed to
|
508
|
+
# create new tables. The default value is `needed`.
|
509
|
+
#
|
510
|
+
# The following values are supported:
|
511
|
+
#
|
512
|
+
# * `needed` - Create the table if it does not exist.
|
513
|
+
# * `never` - The table must already exist. A 'notFound' error is
|
514
|
+
# raised if the table does not exist.
|
515
|
+
#
|
516
|
+
# @!group Attributes
|
517
|
+
def create= value
|
518
|
+
@gapi.configuration.query.create_disposition =
|
519
|
+
Convert.create_disposition value
|
520
|
+
end
|
521
|
+
|
522
|
+
##
|
523
|
+
# Sets the write disposition for when the query results table exists.
|
524
|
+
#
|
525
|
+
# @param [String] value Specifies the action that occurs if the
|
526
|
+
# destination table already exists. The default value is `empty`.
|
527
|
+
#
|
528
|
+
# The following values are supported:
|
529
|
+
#
|
530
|
+
# * `truncate` - BigQuery overwrites the table data.
|
531
|
+
# * `append` - BigQuery appends the data to the table.
|
532
|
+
# * `empty` - A 'duplicate' error is returned in the job result if
|
533
|
+
# the table exists and contains data.
|
534
|
+
#
|
535
|
+
# @!group Attributes
|
536
|
+
def write= value
|
537
|
+
@gapi.configuration.query.write_disposition =
|
538
|
+
Convert.write_disposition value
|
539
|
+
end
|
540
|
+
|
541
|
+
##
|
542
|
+
# Sets the destination for the query results table.
|
543
|
+
#
|
544
|
+
# @param [Table] value The destination table where the query results
|
545
|
+
# should be stored. If not present, a new table will be created
|
546
|
+
# according to the create disposition to store the results.
|
547
|
+
#
|
548
|
+
# @!group Attributes
|
549
|
+
def table= value
|
550
|
+
@gapi.configuration.query.destination_table = table_ref_from value
|
551
|
+
end
|
552
|
+
|
553
|
+
##
|
554
|
+
# Sets the maximum bytes billed for the query.
|
555
|
+
#
|
556
|
+
# @param [Integer] value Limits the bytes billed for this job.
|
557
|
+
# Queries that will have bytes billed beyond this limit will fail
|
558
|
+
# (without incurring a charge). Optional. If unspecified, this will
|
559
|
+
# be set to your project default.
|
560
|
+
#
|
561
|
+
# @!group Attributes
|
562
|
+
def maximum_bytes_billed= value
|
563
|
+
@gapi.configuration.query.maximum_bytes_billed = value
|
564
|
+
end
|
565
|
+
|
566
|
+
##
|
567
|
+
# Sets the labels to use for the job.
|
568
|
+
#
|
569
|
+
# @param [Hash] value A hash of user-provided labels associated with
|
570
|
+
# the job. You can use these to organize and group your jobs. Label
|
571
|
+
# keys and values can be no longer than 63 characters, can only
|
572
|
+
# contain lowercase letters, numeric characters, underscores and
|
573
|
+
# dashes. International characters are allowed. Label values are
|
574
|
+
# optional. Label keys must start with a letter and each label in
|
575
|
+
# the list must have a different key.
|
576
|
+
#
|
577
|
+
# @!group Attributes
|
578
|
+
#
|
579
|
+
def labels= value
|
580
|
+
@gapi.configuration.update! labels: value
|
581
|
+
end
|
582
|
+
|
583
|
+
##
|
584
|
+
# Sets the query syntax to legacy SQL.
|
585
|
+
#
|
586
|
+
# @param [Boolean] value Specifies whether to use BigQuery's [legacy
|
587
|
+
# SQL](https://cloud.google.com/bigquery/docs/reference/legacy-sql)
|
588
|
+
# dialect for this query. If set to false, the query will use
|
589
|
+
# BigQuery's [standard
|
590
|
+
# SQL](https://cloud.google.com/bigquery/docs/reference/standard-sql/)
|
591
|
+
# dialect. Optional. The default value is false.
|
592
|
+
#
|
593
|
+
# @!group Attributes
|
594
|
+
#
|
595
|
+
def legacy_sql= value
|
596
|
+
@gapi.configuration.query.use_legacy_sql = value
|
597
|
+
end
|
598
|
+
|
599
|
+
##
|
600
|
+
# Sets the query syntax to standard SQL.
|
601
|
+
#
|
602
|
+
# @param [Boolean] value Specifies whether to use BigQuery's [standard
|
603
|
+
# SQL](https://cloud.google.com/bigquery/docs/reference/standard-sql/)
|
604
|
+
# dialect for this query. If set to true, the query will use
|
605
|
+
# standard SQL rather than the [legacy
|
606
|
+
# SQL](https://cloud.google.com/bigquery/docs/reference/legacy-sql)
|
607
|
+
# dialect. Optional. The default value is true.
|
608
|
+
#
|
609
|
+
# @!group Attributes
|
610
|
+
#
|
611
|
+
def standard_sql= value
|
612
|
+
@gapi.configuration.query.use_legacy_sql = !value
|
613
|
+
end
|
614
|
+
|
615
|
+
##
|
616
|
+
# Sets definitions for external tables used in the query.
|
617
|
+
#
|
618
|
+
# @param [Hash<String|Symbol, External::DataSource>] value A Hash
|
619
|
+
# that represents the mapping of the external tables to the table
|
620
|
+
# names used in the SQL query. The hash keys are the table names,
|
621
|
+
# and the hash values are the external table objects.
|
622
|
+
#
|
623
|
+
# @!group Attributes
|
624
|
+
#
|
625
|
+
def external= value
|
626
|
+
external_table_pairs = value.map do |name, obj|
|
627
|
+
[String(name), obj.to_gapi]
|
628
|
+
end
|
629
|
+
external_table_hash = Hash[external_table_pairs]
|
630
|
+
@gapi.configuration.query.table_definitions = external_table_hash
|
631
|
+
end
|
632
|
+
|
633
|
+
##
|
634
|
+
# Sets user defined functions for the query.
|
635
|
+
#
|
636
|
+
# @param [Array<String>, String] value User-defined function resources
|
637
|
+
# used in the query. May be either a code resource to load from a
|
638
|
+
# Google Cloud Storage URI (`gs://bucket/path`), or an inline
|
639
|
+
# resource that contains code for a user-defined function (UDF).
|
640
|
+
# Providing an inline code resource is equivalent to providing a URI
|
641
|
+
# for a file containing the same code. See [User-Defined
|
642
|
+
# Functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/user-defined-functions).
|
643
|
+
#
|
644
|
+
# @!group Attributes
|
645
|
+
def udfs= value
|
646
|
+
@gapi.configuration.query.user_defined_function_resources =
|
647
|
+
udfs_gapi_from value
|
648
|
+
end
|
649
|
+
|
650
|
+
##
|
651
|
+
# Sets the encryption configuration of the destination table.
|
652
|
+
#
|
653
|
+
# @param [Google::Cloud::BigQuery::EncryptionConfiguration] val
|
654
|
+
# Custom encryption configuration (e.g., Cloud KMS keys).
|
655
|
+
#
|
656
|
+
# @example
|
657
|
+
# require "google/cloud/bigquery"
|
658
|
+
#
|
659
|
+
# bigquery = Google::Cloud::Bigquery.new
|
660
|
+
# dataset = bigquery.dataset "my_dataset"
|
661
|
+
#
|
662
|
+
# key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
|
663
|
+
# encrypt_config = bigquery.encryption kms_key: key_name
|
664
|
+
# job = bigquery.query_job "SELECT 1;" do |query|
|
665
|
+
# query.table = dataset.table "my_table", skip_lookup: true
|
666
|
+
# query.encryption = encrypt_config
|
667
|
+
# end
|
668
|
+
#
|
669
|
+
# @!group Attributes
|
670
|
+
def encryption= val
|
671
|
+
@gapi.configuration.query.update!(
|
672
|
+
destination_encryption_configuration: val.to_gapi
|
673
|
+
)
|
674
|
+
end
|
675
|
+
|
676
|
+
##
|
677
|
+
# @private Returns the Google API client library version of this job.
|
678
|
+
#
|
679
|
+
# @return [<Google::Apis::BigqueryV2::Job>] (See
|
680
|
+
# {Google::Apis::BigqueryV2::Job})
|
681
|
+
def to_gapi
|
682
|
+
@gapi
|
683
|
+
end
|
684
|
+
|
685
|
+
protected
|
686
|
+
|
687
|
+
# Creates a table reference from a table object.
|
688
|
+
def table_ref_from tbl
|
689
|
+
return nil if tbl.nil?
|
690
|
+
Google::Apis::BigqueryV2::TableReference.new(
|
691
|
+
project_id: tbl.project_id,
|
692
|
+
dataset_id: tbl.dataset_id,
|
693
|
+
table_id: tbl.table_id
|
694
|
+
)
|
695
|
+
end
|
696
|
+
|
697
|
+
def priority_value str
|
698
|
+
{ "batch" => "BATCH",
|
699
|
+
"interactive" => "INTERACTIVE" }[str.to_s.downcase]
|
700
|
+
end
|
701
|
+
|
702
|
+
def udfs_gapi_from array_or_str
|
703
|
+
Array(array_or_str).map do |uri_or_code|
|
704
|
+
resource =
|
705
|
+
Google::Apis::BigqueryV2::UserDefinedFunctionResource.new
|
706
|
+
if uri_or_code.start_with?("gs://")
|
707
|
+
resource.resource_uri = uri_or_code
|
708
|
+
else
|
709
|
+
resource.inline_code = uri_or_code
|
710
|
+
end
|
711
|
+
resource
|
712
|
+
end
|
713
|
+
end
|
714
|
+
end
|
715
|
+
|
316
716
|
##
|
317
717
|
# Represents a stage in the execution plan for the query.
|
318
718
|
#
|
@@ -452,7 +852,8 @@ module Google
|
|
452
852
|
def ensure_schema!
|
453
853
|
return unless destination_schema.nil?
|
454
854
|
|
455
|
-
query_results_gapi = service.job_query_results
|
855
|
+
query_results_gapi = service.job_query_results \
|
856
|
+
job_id, location: location, max: 0
|
456
857
|
# raise "unable to retrieve schema" if query_results_gapi.schema.nil?
|
457
858
|
@destination_schema_gapi = query_results_gapi.schema
|
458
859
|
end
|