google-cloud-bigquery 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +27 -19
- data/lib/google/cloud/bigquery/convert.rb +106 -13
- data/lib/google/cloud/bigquery/copy_job.rb +168 -0
- data/lib/google/cloud/bigquery/dataset.rb +313 -112
- data/lib/google/cloud/bigquery/encryption_configuration.rb +124 -0
- data/lib/google/cloud/bigquery/extract_job.rb +149 -0
- data/lib/google/cloud/bigquery/job.rb +14 -3
- data/lib/google/cloud/bigquery/load_job.rb +770 -0
- data/lib/google/cloud/bigquery/project.rb +118 -20
- data/lib/google/cloud/bigquery/query_job.rb +403 -2
- data/lib/google/cloud/bigquery/schema/field.rb +22 -22
- data/lib/google/cloud/bigquery/service.rb +45 -328
- data/lib/google/cloud/bigquery/table.rb +375 -95
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +3 -2
@@ -108,6 +108,9 @@ module Google
|
|
108
108
|
# See [Data Types](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types)
|
109
109
|
# for an overview of each BigQuery data type, including allowed values.
|
110
110
|
#
|
111
|
+
# The geographic location for the job ("US", "EU", etc.) can be set via
|
112
|
+
# {QueryJob::Updater#location=} in a block passed to this method.
|
113
|
+
#
|
111
114
|
# @param [String] query A query string, following the BigQuery [query
|
112
115
|
# syntax](https://cloud.google.com/bigquery/query-reference), of the
|
113
116
|
# query to execute. Example: "SELECT count(f1) FROM
|
@@ -178,11 +181,6 @@ module Google
|
|
178
181
|
# Flattens all nested and repeated fields in the query results. The
|
179
182
|
# default value is `true`. `large_results` parameter must be `true` if
|
180
183
|
# this is set to `false`.
|
181
|
-
# @param [Integer] maximum_billing_tier Limits the billing tier for this
|
182
|
-
# job. Queries that have resource usage beyond this tier will fail
|
183
|
-
# (without incurring a charge). Optional. If unspecified, this will be
|
184
|
-
# set to your project default. For more information, see [High-Compute
|
185
|
-
# queries](https://cloud.google.com/bigquery/pricing#high-compute).
|
186
184
|
# @param [Integer] maximum_bytes_billed Limits the bytes billed for this
|
187
185
|
# job. Queries that will have bytes billed beyond this limit will fail
|
188
186
|
# (without incurring a charge). Optional. If unspecified, this will be
|
@@ -219,6 +217,11 @@ module Google
|
|
219
217
|
# inline code resource is equivalent to providing a URI for a file
|
220
218
|
# containing the same code. See [User-Defined
|
221
219
|
# Functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/user-defined-functions).
|
220
|
+
# @param [Integer] maximum_billing_tier Deprecated: Change the billing
|
221
|
+
# tier to allow high-compute queries.
|
222
|
+
# @yield [job] a job configuration object
|
223
|
+
# @yieldparam [Google::Cloud::Bigquery::QueryJob::Updater] job a job
|
224
|
+
# configuration object for setting query options.
|
222
225
|
#
|
223
226
|
# @return [Google::Cloud::Bigquery::QueryJob]
|
224
227
|
#
|
@@ -243,7 +246,7 @@ module Google
|
|
243
246
|
# bigquery = Google::Cloud::Bigquery.new
|
244
247
|
#
|
245
248
|
# job = bigquery.query_job "SELECT name FROM " \
|
246
|
-
# "[my_project:my_dataset.my_table]",
|
249
|
+
# " [my_project:my_dataset.my_table]",
|
247
250
|
# legacy_sql: true
|
248
251
|
#
|
249
252
|
# job.wait_until_done!
|
@@ -287,7 +290,7 @@ module Google
|
|
287
290
|
# end
|
288
291
|
# end
|
289
292
|
#
|
290
|
-
# @example Query using external data source:
|
293
|
+
# @example Query using external data source, set destination:
|
291
294
|
# require "google/cloud/bigquery"
|
292
295
|
#
|
293
296
|
# bigquery = Google::Cloud::Bigquery.new
|
@@ -298,8 +301,11 @@ module Google
|
|
298
301
|
# csv.skip_leading_rows = 1
|
299
302
|
# end
|
300
303
|
#
|
301
|
-
# job = bigquery.query_job "SELECT * FROM my_ext_table"
|
302
|
-
#
|
304
|
+
# job = bigquery.query_job "SELECT * FROM my_ext_table" do |query|
|
305
|
+
# query.external = { my_ext_table: csv_table }
|
306
|
+
# dataset = bigquery.dataset "my_dataset", skip_lookup: true
|
307
|
+
# query.table = dataset.table "my_table", skip_lookup: true
|
308
|
+
# end
|
303
309
|
#
|
304
310
|
# job.wait_until_done!
|
305
311
|
# if !job.failed?
|
@@ -319,13 +325,18 @@ module Google
|
|
319
325
|
options = { priority: priority, cache: cache, table: table,
|
320
326
|
create: create, write: write,
|
321
327
|
large_results: large_results, flatten: flatten,
|
322
|
-
dataset: dataset, project: project,
|
328
|
+
dataset: dataset, project: (project || self.project),
|
323
329
|
legacy_sql: legacy_sql, standard_sql: standard_sql,
|
324
330
|
maximum_billing_tier: maximum_billing_tier,
|
325
331
|
maximum_bytes_billed: maximum_bytes_billed,
|
326
|
-
|
327
|
-
|
328
|
-
|
332
|
+
external: external, job_id: job_id, prefix: prefix,
|
333
|
+
labels: labels, udfs: udfs, params: params }
|
334
|
+
|
335
|
+
updater = QueryJob::Updater.from_options service, query, options
|
336
|
+
|
337
|
+
yield updater if block_given?
|
338
|
+
|
339
|
+
gapi = service.query_job updater.to_gapi
|
329
340
|
Job.from_gapi gapi, service
|
330
341
|
end
|
331
342
|
|
@@ -355,6 +366,9 @@ module Google
|
|
355
366
|
# See [Data Types](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types)
|
356
367
|
# for an overview of each BigQuery data type, including allowed values.
|
357
368
|
#
|
369
|
+
# The geographic location for the job ("US", "EU", etc.) can be set via
|
370
|
+
# {QueryJob::Updater#location=} in a block passed to this method.
|
371
|
+
#
|
358
372
|
# @see https://cloud.google.com/bigquery/querying-data Querying Data
|
359
373
|
#
|
360
374
|
# @param [String] query A query string, following the BigQuery [query
|
@@ -409,6 +423,9 @@ module Google
|
|
409
423
|
# When set to false, the values of `large_results` and `flatten` are
|
410
424
|
# ignored; the query will be run as if `large_results` is true and
|
411
425
|
# `flatten` is false. Optional. The default value is false.
|
426
|
+
# @yield [job] a job configuration object
|
427
|
+
# @yieldparam [Google::Cloud::Bigquery::QueryJob::Updater] job a job
|
428
|
+
# configuration object for setting additional options for the query.
|
412
429
|
#
|
413
430
|
# @return [Google::Cloud::Bigquery::Data]
|
414
431
|
#
|
@@ -475,7 +492,7 @@ module Google
|
|
475
492
|
# puts row[:name]
|
476
493
|
# end
|
477
494
|
#
|
478
|
-
# @example Query using external data source:
|
495
|
+
# @example Query using external data source, set destination:
|
479
496
|
# require "google/cloud/bigquery"
|
480
497
|
#
|
481
498
|
# bigquery = Google::Cloud::Bigquery.new
|
@@ -486,8 +503,11 @@ module Google
|
|
486
503
|
# csv.skip_leading_rows = 1
|
487
504
|
# end
|
488
505
|
#
|
489
|
-
# data = bigquery.query "SELECT * FROM my_ext_table"
|
490
|
-
#
|
506
|
+
# data = bigquery.query "SELECT * FROM my_ext_table" do |query|
|
507
|
+
# query.external = { my_ext_table: csv_table }
|
508
|
+
# dataset = bigquery.dataset "my_dataset", skip_lookup: true
|
509
|
+
# query.table = dataset.table "my_table", skip_lookup: true
|
510
|
+
# end
|
491
511
|
#
|
492
512
|
# data.each do |row|
|
493
513
|
# puts row[:name]
|
@@ -496,11 +516,16 @@ module Google
|
|
496
516
|
def query query, params: nil, external: nil, max: nil, cache: true,
|
497
517
|
dataset: nil, project: nil, standard_sql: nil, legacy_sql: nil
|
498
518
|
ensure_service!
|
499
|
-
options = { cache: cache, dataset: dataset,
|
519
|
+
options = { priority: "INTERACTIVE", cache: cache, dataset: dataset,
|
520
|
+
project: project || self.project,
|
500
521
|
legacy_sql: legacy_sql, standard_sql: standard_sql,
|
501
522
|
params: params, external: external }
|
523
|
+
updater = QueryJob::Updater.from_options service, query, options
|
524
|
+
|
525
|
+
yield updater if block_given?
|
502
526
|
|
503
|
-
|
527
|
+
gapi = service.query_job updater.to_gapi
|
528
|
+
job = Job.from_gapi gapi, service
|
504
529
|
job.wait_until_done!
|
505
530
|
|
506
531
|
if job.failed?
|
@@ -734,6 +759,8 @@ module Google
|
|
734
759
|
# Retrieves an existing job by ID.
|
735
760
|
#
|
736
761
|
# @param [String] job_id The ID of a job.
|
762
|
+
# @param [String] location The geographic location where the job was
|
763
|
+
# created. Required except for US and EU.
|
737
764
|
#
|
738
765
|
# @return [Google::Cloud::Bigquery::Job, nil] Returns `nil` if the job
|
739
766
|
# does not exist.
|
@@ -745,9 +772,9 @@ module Google
|
|
745
772
|
#
|
746
773
|
# job = bigquery.job "my_job"
|
747
774
|
#
|
748
|
-
def job job_id
|
775
|
+
def job job_id, location: nil
|
749
776
|
ensure_service!
|
750
|
-
gapi = service.get_job job_id
|
777
|
+
gapi = service.get_job job_id, location: location
|
751
778
|
Job.from_gapi gapi, service
|
752
779
|
rescue Google::Cloud::NotFoundError
|
753
780
|
nil
|
@@ -941,6 +968,77 @@ module Google
|
|
941
968
|
s
|
942
969
|
end
|
943
970
|
|
971
|
+
##
|
972
|
+
# Creates a new Bigquery::EncryptionConfiguration instance.
|
973
|
+
#
|
974
|
+
# This method does not execute an API call. Use the encryption
|
975
|
+
# configuration to encrypt a table when creating one via
|
976
|
+
# Bigquery::Dataset#create_table, Bigquery::Dataset#load,
|
977
|
+
# Bigquery::Table#copy, or Bigquery::Project#query.
|
978
|
+
#
|
979
|
+
# @param [String] kms_key Name of the Cloud KMS encryption key that
|
980
|
+
# will be used to protect the destination BigQuery table. The BigQuery
|
981
|
+
# Service Account associated with your project requires access to this
|
982
|
+
# encryption key.
|
983
|
+
#
|
984
|
+
# @example Encrypt a new table
|
985
|
+
# require "google/cloud/bigquery"
|
986
|
+
#
|
987
|
+
# bigquery = Google::Cloud::Bigquery.new
|
988
|
+
# dataset = bigquery.dataset "my_dataset"
|
989
|
+
#
|
990
|
+
# key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
|
991
|
+
# encrypt_config = bigquery.encryption kms_key: key_name
|
992
|
+
#
|
993
|
+
# table = dataset.create_table "my_table" do |updater|
|
994
|
+
# updater.encryption = encrypt_config
|
995
|
+
# end
|
996
|
+
#
|
997
|
+
# @example Encrypt a load destination table
|
998
|
+
# require "google/cloud/bigquery"
|
999
|
+
#
|
1000
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1001
|
+
# dataset = bigquery.dataset "my_dataset"
|
1002
|
+
#
|
1003
|
+
# key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
|
1004
|
+
# encrypt_config = bigquery.encryption kms_key: key_name
|
1005
|
+
# job = dataset.load_job "my_table", "gs://abc/file" do |job|
|
1006
|
+
# job.encryption = encrypt_config
|
1007
|
+
# end
|
1008
|
+
#
|
1009
|
+
# @example Encrypt a copy destination table
|
1010
|
+
# require "google/cloud/bigquery"
|
1011
|
+
#
|
1012
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1013
|
+
# dataset = bigquery.dataset "my_dataset"
|
1014
|
+
# table = dataset.table "my_table"
|
1015
|
+
#
|
1016
|
+
# key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
|
1017
|
+
# encrypt_config = bigquery.encryption kms_key: key_name
|
1018
|
+
# job = table.copy_job "my_dataset.new_table" do |job|
|
1019
|
+
# job.encryption = encrypt_config
|
1020
|
+
# end
|
1021
|
+
#
|
1022
|
+
# @example Encrypt a query destination table
|
1023
|
+
# require "google/cloud/bigquery"
|
1024
|
+
#
|
1025
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1026
|
+
# dataset = bigquery.dataset "my_dataset"
|
1027
|
+
#
|
1028
|
+
# key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
|
1029
|
+
# encrypt_config = bigquery.encryption kms_key: key_name
|
1030
|
+
# job = bigquery.query_job "SELECT 1;" do |query|
|
1031
|
+
# query.table = dataset.table "my_table", skip_lookup: true
|
1032
|
+
# query.encryption = encrypt_config
|
1033
|
+
# end
|
1034
|
+
#
|
1035
|
+
# @return [Google::Cloud::Bigquery::EncryptionConfiguration]
|
1036
|
+
def encryption kms_key: nil
|
1037
|
+
encrypt_config = Bigquery::EncryptionConfiguration.new
|
1038
|
+
encrypt_config.kms_key = kms_key unless kms_key.nil?
|
1039
|
+
encrypt_config
|
1040
|
+
end
|
1041
|
+
|
944
1042
|
##
|
945
1043
|
# @private New Project from a Google API Client object, using the
|
946
1044
|
# same Credentials as this project.
|
@@ -15,6 +15,8 @@
|
|
15
15
|
|
16
16
|
require "google/cloud/bigquery/service"
|
17
17
|
require "google/cloud/bigquery/data"
|
18
|
+
require "google/cloud/bigquery/encryption_configuration"
|
19
|
+
require "google/apis/bigquery_v2"
|
18
20
|
|
19
21
|
module Google
|
20
22
|
module Cloud
|
@@ -244,6 +246,19 @@ module Google
|
|
244
246
|
end
|
245
247
|
end
|
246
248
|
|
249
|
+
##
|
250
|
+
# The encryption configuration of the destination table.
|
251
|
+
#
|
252
|
+
# @return [Google::Cloud::BigQuery::EncryptionConfiguration] Custom
|
253
|
+
# encryption configuration (e.g., Cloud KMS keys).
|
254
|
+
#
|
255
|
+
# @!group Attributes
|
256
|
+
def encryption
|
257
|
+
EncryptionConfiguration.from_gapi(
|
258
|
+
@gapi.configuration.query.destination_encryption_configuration
|
259
|
+
)
|
260
|
+
end
|
261
|
+
|
247
262
|
##
|
248
263
|
# Refreshes the job until the job is `DONE`.
|
249
264
|
# The delay between refreshes will incrementally increase.
|
@@ -264,7 +279,8 @@ module Google
|
|
264
279
|
|
265
280
|
ensure_service!
|
266
281
|
loop do
|
267
|
-
query_results_gapi = service.job_query_results
|
282
|
+
query_results_gapi = service.job_query_results \
|
283
|
+
job_id, location: location, max: 0
|
268
284
|
if query_results_gapi.job_complete
|
269
285
|
@destination_schema_gapi = query_results_gapi.schema
|
270
286
|
break
|
@@ -313,6 +329,390 @@ module Google
|
|
313
329
|
end
|
314
330
|
alias query_results data
|
315
331
|
|
332
|
+
##
|
333
|
+
# Yielded to a block to accumulate changes for a patch request.
|
334
|
+
class Updater < QueryJob
|
335
|
+
##
|
336
|
+
# @private Create an Updater object.
|
337
|
+
def initialize service, gapi
|
338
|
+
@service = service
|
339
|
+
@gapi = gapi
|
340
|
+
end
|
341
|
+
|
342
|
+
# rubocop:disable all
|
343
|
+
|
344
|
+
##
|
345
|
+
# @private Create an Updater from an options hash.
|
346
|
+
#
|
347
|
+
# @return [Google::Cloud::Bigquery::QueryJob::Updater] A job
|
348
|
+
# configuration object for setting query options.
|
349
|
+
def self.from_options service, query, options
|
350
|
+
job_ref = service.job_ref_from options[:job_id], options[:prefix]
|
351
|
+
dataset_config = service.dataset_ref_from options[:dataset],
|
352
|
+
options[:project]
|
353
|
+
req = Google::Apis::BigqueryV2::Job.new(
|
354
|
+
job_reference: job_ref,
|
355
|
+
configuration: Google::Apis::BigqueryV2::JobConfiguration.new(
|
356
|
+
query: Google::Apis::BigqueryV2::JobConfigurationQuery.new(
|
357
|
+
query: query,
|
358
|
+
default_dataset: dataset_config,
|
359
|
+
maximum_billing_tier: options[:maximum_billing_tier]
|
360
|
+
)
|
361
|
+
)
|
362
|
+
)
|
363
|
+
|
364
|
+
updater = QueryJob::Updater.new service, req
|
365
|
+
updater.params = options[:params] if options[:params]
|
366
|
+
updater.create = options[:create]
|
367
|
+
updater.write = options[:write]
|
368
|
+
updater.table = options[:table]
|
369
|
+
updater.maximum_bytes_billed = options[:maximum_bytes_billed]
|
370
|
+
updater.labels = options[:labels] if options[:labels]
|
371
|
+
updater.legacy_sql = Convert.resolve_legacy_sql(
|
372
|
+
options[:standard_sql], options[:legacy_sql])
|
373
|
+
updater.external = options[:external] if options[:external]
|
374
|
+
updater.priority = options[:priority]
|
375
|
+
updater.cache = options[:cache]
|
376
|
+
updater.large_results = options[:large_results]
|
377
|
+
updater.flatten = options[:flatten]
|
378
|
+
updater.udfs = options[:udfs]
|
379
|
+
updater
|
380
|
+
end
|
381
|
+
|
382
|
+
# rubocop:enable all
|
383
|
+
|
384
|
+
##
|
385
|
+
# Sets the geographic location where the job should run. Required
|
386
|
+
# except for US and EU.
|
387
|
+
#
|
388
|
+
# @param [String] value A geographic location, such as "US", "EU" or
|
389
|
+
# "asia-northeast1". Required except for US and EU.
|
390
|
+
#
|
391
|
+
# @example
|
392
|
+
# require "google/cloud/bigquery"
|
393
|
+
#
|
394
|
+
# bigquery = Google::Cloud::Bigquery.new
|
395
|
+
# dataset = bigquery.dataset "my_dataset"
|
396
|
+
#
|
397
|
+
# job = bigquery.query_job "SELECT 1;" do |query|
|
398
|
+
# query.table = dataset.table "my_table", skip_lookup: true
|
399
|
+
# query.location = "EU"
|
400
|
+
# end
|
401
|
+
#
|
402
|
+
# @!group Attributes
|
403
|
+
def location= value
|
404
|
+
@gapi.job_reference.location = value
|
405
|
+
end
|
406
|
+
|
407
|
+
##
|
408
|
+
# Sets the priority of the query.
|
409
|
+
#
|
410
|
+
# @param [String] value Specifies a priority for the query. Possible
|
411
|
+
# values include `INTERACTIVE` and `BATCH`.
|
412
|
+
#
|
413
|
+
# @!group Attributes
|
414
|
+
def priority= value
|
415
|
+
@gapi.configuration.query.priority = priority_value value
|
416
|
+
end
|
417
|
+
|
418
|
+
##
|
419
|
+
# Specifies to look in the query cache for results.
|
420
|
+
#
|
421
|
+
# @param [Boolean] value Whether to look for the result in the query
|
422
|
+
# cache. The query cache is a best-effort cache that will be flushed
|
423
|
+
# whenever tables in the query are modified. The default value is
|
424
|
+
# true. For more information, see [query
|
425
|
+
# caching](https://developers.google.com/bigquery/querying-data).
|
426
|
+
#
|
427
|
+
# @!group Attributes
|
428
|
+
def cache= value
|
429
|
+
@gapi.configuration.query.use_query_cache = value
|
430
|
+
end
|
431
|
+
|
432
|
+
##
|
433
|
+
# Allow large results for a legacy SQL query.
|
434
|
+
#
|
435
|
+
# @param [Boolean] value This option is specific to Legacy SQL.
|
436
|
+
# If `true`, allows the query to produce arbitrarily large result
|
437
|
+
# tables at a slight cost in performance. Requires `table` parameter
|
438
|
+
# to be set.
|
439
|
+
#
|
440
|
+
# @!group Attributes
|
441
|
+
def large_results= value
|
442
|
+
@gapi.configuration.query.allow_large_results = value
|
443
|
+
end
|
444
|
+
|
445
|
+
##
|
446
|
+
# Flatten nested and repeated fields in legacy SQL queries.
|
447
|
+
#
|
448
|
+
# @param [Boolean] value This option is specific to Legacy SQL.
|
449
|
+
# Flattens all nested and repeated fields in the query results. The
|
450
|
+
# default value is `true`. `large_results` parameter must be `true`
|
451
|
+
# if this is set to `false`.
|
452
|
+
#
|
453
|
+
# @!group Attributes
|
454
|
+
def flatten= value
|
455
|
+
@gapi.configuration.query.flatten_results = value
|
456
|
+
end
|
457
|
+
|
458
|
+
##
|
459
|
+
# Sets the default dataset of tables referenced in the query.
|
460
|
+
#
|
461
|
+
# @param [Dataset] value The default dataset to use for unqualified
|
462
|
+
# table names in the query.
|
463
|
+
#
|
464
|
+
# @!group Attributes
|
465
|
+
def dataset= value
|
466
|
+
@gapi.configuration.query.default_dataset =
|
467
|
+
@service.dataset_ref_from value
|
468
|
+
end
|
469
|
+
|
470
|
+
##
|
471
|
+
# Sets the query parameters. Standard SQL only.
|
472
|
+
#
|
473
|
+
# @param [Array, Hash] params Used to pass query arguments when the
|
474
|
+
# `query` string contains either positional (`?`) or named
|
475
|
+
# (`@myparam`) query parameters. If value passed is an array
|
476
|
+
# `["foo"]`, the query must use positional query parameters. If
|
477
|
+
# value passed is a hash `{ myparam: "foo" }`, the query must use
|
478
|
+
# named query parameters. When set, `legacy_sql` will automatically
|
479
|
+
# be set to false and `standard_sql` to true.
|
480
|
+
#
|
481
|
+
# @!group Attributes
|
482
|
+
def params= params
|
483
|
+
case params
|
484
|
+
when Array then
|
485
|
+
@gapi.configuration.query.use_legacy_sql = false
|
486
|
+
@gapi.configuration.query.parameter_mode = "POSITIONAL"
|
487
|
+
@gapi.configuration.query.query_parameters = params.map do |param|
|
488
|
+
Convert.to_query_param param
|
489
|
+
end
|
490
|
+
when Hash then
|
491
|
+
@gapi.configuration.query.use_legacy_sql = false
|
492
|
+
@gapi.configuration.query.parameter_mode = "NAMED"
|
493
|
+
@gapi.configuration.query.query_parameters =
|
494
|
+
params.map do |name, param|
|
495
|
+
Convert.to_query_param(param).tap do |named_param|
|
496
|
+
named_param.name = String name
|
497
|
+
end
|
498
|
+
end
|
499
|
+
else
|
500
|
+
raise "Query parameters must be an Array or a Hash."
|
501
|
+
end
|
502
|
+
end
|
503
|
+
|
504
|
+
##
|
505
|
+
# Sets the create disposition for creating the query results table.
|
506
|
+
#
|
507
|
+
# @param [String] value Specifies whether the job is allowed to
|
508
|
+
# create new tables. The default value is `needed`.
|
509
|
+
#
|
510
|
+
# The following values are supported:
|
511
|
+
#
|
512
|
+
# * `needed` - Create the table if it does not exist.
|
513
|
+
# * `never` - The table must already exist. A 'notFound' error is
|
514
|
+
# raised if the table does not exist.
|
515
|
+
#
|
516
|
+
# @!group Attributes
|
517
|
+
def create= value
|
518
|
+
@gapi.configuration.query.create_disposition =
|
519
|
+
Convert.create_disposition value
|
520
|
+
end
|
521
|
+
|
522
|
+
##
|
523
|
+
# Sets the write disposition for when the query results table exists.
|
524
|
+
#
|
525
|
+
# @param [String] value Specifies the action that occurs if the
|
526
|
+
# destination table already exists. The default value is `empty`.
|
527
|
+
#
|
528
|
+
# The following values are supported:
|
529
|
+
#
|
530
|
+
# * `truncate` - BigQuery overwrites the table data.
|
531
|
+
# * `append` - BigQuery appends the data to the table.
|
532
|
+
# * `empty` - A 'duplicate' error is returned in the job result if
|
533
|
+
# the table exists and contains data.
|
534
|
+
#
|
535
|
+
# @!group Attributes
|
536
|
+
def write= value
|
537
|
+
@gapi.configuration.query.write_disposition =
|
538
|
+
Convert.write_disposition value
|
539
|
+
end
|
540
|
+
|
541
|
+
##
|
542
|
+
# Sets the destination for the query results table.
|
543
|
+
#
|
544
|
+
# @param [Table] value The destination table where the query results
|
545
|
+
# should be stored. If not present, a new table will be created
|
546
|
+
# according to the create disposition to store the results.
|
547
|
+
#
|
548
|
+
# @!group Attributes
|
549
|
+
def table= value
|
550
|
+
@gapi.configuration.query.destination_table = table_ref_from value
|
551
|
+
end
|
552
|
+
|
553
|
+
##
|
554
|
+
# Sets the maximum bytes billed for the query.
|
555
|
+
#
|
556
|
+
# @param [Integer] value Limits the bytes billed for this job.
|
557
|
+
# Queries that will have bytes billed beyond this limit will fail
|
558
|
+
# (without incurring a charge). Optional. If unspecified, this will
|
559
|
+
# be set to your project default.
|
560
|
+
#
|
561
|
+
# @!group Attributes
|
562
|
+
def maximum_bytes_billed= value
|
563
|
+
@gapi.configuration.query.maximum_bytes_billed = value
|
564
|
+
end
|
565
|
+
|
566
|
+
##
|
567
|
+
# Sets the labels to use for the job.
|
568
|
+
#
|
569
|
+
# @param [Hash] value A hash of user-provided labels associated with
|
570
|
+
# the job. You can use these to organize and group your jobs. Label
|
571
|
+
# keys and values can be no longer than 63 characters, can only
|
572
|
+
# contain lowercase letters, numeric characters, underscores and
|
573
|
+
# dashes. International characters are allowed. Label values are
|
574
|
+
# optional. Label keys must start with a letter and each label in
|
575
|
+
# the list must have a different key.
|
576
|
+
#
|
577
|
+
# @!group Attributes
|
578
|
+
#
|
579
|
+
def labels= value
|
580
|
+
@gapi.configuration.update! labels: value
|
581
|
+
end
|
582
|
+
|
583
|
+
##
|
584
|
+
# Sets the query syntax to legacy SQL.
|
585
|
+
#
|
586
|
+
# @param [Boolean] value Specifies whether to use BigQuery's [legacy
|
587
|
+
# SQL](https://cloud.google.com/bigquery/docs/reference/legacy-sql)
|
588
|
+
# dialect for this query. If set to false, the query will use
|
589
|
+
# BigQuery's [standard
|
590
|
+
# SQL](https://cloud.google.com/bigquery/docs/reference/standard-sql/)
|
591
|
+
# dialect. Optional. The default value is false.
|
592
|
+
#
|
593
|
+
# @!group Attributes
|
594
|
+
#
|
595
|
+
def legacy_sql= value
|
596
|
+
@gapi.configuration.query.use_legacy_sql = value
|
597
|
+
end
|
598
|
+
|
599
|
+
##
|
600
|
+
# Sets the query syntax to standard SQL.
|
601
|
+
#
|
602
|
+
# @param [Boolean] value Specifies whether to use BigQuery's [standard
|
603
|
+
# SQL](https://cloud.google.com/bigquery/docs/reference/standard-sql/)
|
604
|
+
# dialect for this query. If set to true, the query will use
|
605
|
+
# standard SQL rather than the [legacy
|
606
|
+
# SQL](https://cloud.google.com/bigquery/docs/reference/legacy-sql)
|
607
|
+
# dialect. Optional. The default value is true.
|
608
|
+
#
|
609
|
+
# @!group Attributes
|
610
|
+
#
|
611
|
+
def standard_sql= value
|
612
|
+
@gapi.configuration.query.use_legacy_sql = !value
|
613
|
+
end
|
614
|
+
|
615
|
+
##
|
616
|
+
# Sets definitions for external tables used in the query.
|
617
|
+
#
|
618
|
+
# @param [Hash<String|Symbol, External::DataSource>] value A Hash
|
619
|
+
# that represents the mapping of the external tables to the table
|
620
|
+
# names used in the SQL query. The hash keys are the table names,
|
621
|
+
# and the hash values are the external table objects.
|
622
|
+
#
|
623
|
+
# @!group Attributes
|
624
|
+
#
|
625
|
+
def external= value
|
626
|
+
external_table_pairs = value.map do |name, obj|
|
627
|
+
[String(name), obj.to_gapi]
|
628
|
+
end
|
629
|
+
external_table_hash = Hash[external_table_pairs]
|
630
|
+
@gapi.configuration.query.table_definitions = external_table_hash
|
631
|
+
end
|
632
|
+
|
633
|
+
##
|
634
|
+
# Sets user defined functions for the query.
|
635
|
+
#
|
636
|
+
# @param [Array<String>, String] value User-defined function resources
|
637
|
+
# used in the query. May be either a code resource to load from a
|
638
|
+
# Google Cloud Storage URI (`gs://bucket/path`), or an inline
|
639
|
+
# resource that contains code for a user-defined function (UDF).
|
640
|
+
# Providing an inline code resource is equivalent to providing a URI
|
641
|
+
# for a file containing the same code. See [User-Defined
|
642
|
+
# Functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/user-defined-functions).
|
643
|
+
#
|
644
|
+
# @!group Attributes
|
645
|
+
def udfs= value
|
646
|
+
@gapi.configuration.query.user_defined_function_resources =
|
647
|
+
udfs_gapi_from value
|
648
|
+
end
|
649
|
+
|
650
|
+
##
|
651
|
+
# Sets the encryption configuration of the destination table.
|
652
|
+
#
|
653
|
+
# @param [Google::Cloud::BigQuery::EncryptionConfiguration] val
|
654
|
+
# Custom encryption configuration (e.g., Cloud KMS keys).
|
655
|
+
#
|
656
|
+
# @example
|
657
|
+
# require "google/cloud/bigquery"
|
658
|
+
#
|
659
|
+
# bigquery = Google::Cloud::Bigquery.new
|
660
|
+
# dataset = bigquery.dataset "my_dataset"
|
661
|
+
#
|
662
|
+
# key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
|
663
|
+
# encrypt_config = bigquery.encryption kms_key: key_name
|
664
|
+
# job = bigquery.query_job "SELECT 1;" do |query|
|
665
|
+
# query.table = dataset.table "my_table", skip_lookup: true
|
666
|
+
# query.encryption = encrypt_config
|
667
|
+
# end
|
668
|
+
#
|
669
|
+
# @!group Attributes
|
670
|
+
def encryption= val
|
671
|
+
@gapi.configuration.query.update!(
|
672
|
+
destination_encryption_configuration: val.to_gapi
|
673
|
+
)
|
674
|
+
end
|
675
|
+
|
676
|
+
##
|
677
|
+
# @private Returns the Google API client library version of this job.
|
678
|
+
#
|
679
|
+
# @return [<Google::Apis::BigqueryV2::Job>] (See
|
680
|
+
# {Google::Apis::BigqueryV2::Job})
|
681
|
+
def to_gapi
|
682
|
+
@gapi
|
683
|
+
end
|
684
|
+
|
685
|
+
protected
|
686
|
+
|
687
|
+
# Creates a table reference from a table object.
|
688
|
+
def table_ref_from tbl
|
689
|
+
return nil if tbl.nil?
|
690
|
+
Google::Apis::BigqueryV2::TableReference.new(
|
691
|
+
project_id: tbl.project_id,
|
692
|
+
dataset_id: tbl.dataset_id,
|
693
|
+
table_id: tbl.table_id
|
694
|
+
)
|
695
|
+
end
|
696
|
+
|
697
|
+
def priority_value str
|
698
|
+
{ "batch" => "BATCH",
|
699
|
+
"interactive" => "INTERACTIVE" }[str.to_s.downcase]
|
700
|
+
end
|
701
|
+
|
702
|
+
def udfs_gapi_from array_or_str
|
703
|
+
Array(array_or_str).map do |uri_or_code|
|
704
|
+
resource =
|
705
|
+
Google::Apis::BigqueryV2::UserDefinedFunctionResource.new
|
706
|
+
if uri_or_code.start_with?("gs://")
|
707
|
+
resource.resource_uri = uri_or_code
|
708
|
+
else
|
709
|
+
resource.inline_code = uri_or_code
|
710
|
+
end
|
711
|
+
resource
|
712
|
+
end
|
713
|
+
end
|
714
|
+
end
|
715
|
+
|
316
716
|
##
|
317
717
|
# Represents a stage in the execution plan for the query.
|
318
718
|
#
|
@@ -452,7 +852,8 @@ module Google
|
|
452
852
|
def ensure_schema!
|
453
853
|
return unless destination_schema.nil?
|
454
854
|
|
455
|
-
query_results_gapi = service.job_query_results
|
855
|
+
query_results_gapi = service.job_query_results \
|
856
|
+
job_id, location: location, max: 0
|
456
857
|
# raise "unable to retrieve schema" if query_results_gapi.schema.nil?
|
457
858
|
@destination_schema_gapi = query_results_gapi.schema
|
458
859
|
end
|