google-cloud-bigquery 1.18.1 → 1.21.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/TROUBLESHOOTING.md +2 -8
- data/lib/google-cloud-bigquery.rb +8 -2
- data/lib/google/cloud/bigquery/argument.rb +197 -0
- data/lib/google/cloud/bigquery/copy_job.rb +18 -1
- data/lib/google/cloud/bigquery/data.rb +27 -0
- data/lib/google/cloud/bigquery/dataset.rb +397 -49
- data/lib/google/cloud/bigquery/dataset/list.rb +1 -2
- data/lib/google/cloud/bigquery/external.rb +24 -0
- data/lib/google/cloud/bigquery/extract_job.rb +19 -2
- data/lib/google/cloud/bigquery/job.rb +198 -0
- data/lib/google/cloud/bigquery/job/list.rb +5 -5
- data/lib/google/cloud/bigquery/load_job.rb +273 -26
- data/lib/google/cloud/bigquery/model.rb +6 -4
- data/lib/google/cloud/bigquery/project.rb +109 -22
- data/lib/google/cloud/bigquery/project/list.rb +1 -2
- data/lib/google/cloud/bigquery/query_job.rb +295 -0
- data/lib/google/cloud/bigquery/routine.rb +1108 -0
- data/lib/google/cloud/bigquery/routine/list.rb +165 -0
- data/lib/google/cloud/bigquery/schema.rb +2 -2
- data/lib/google/cloud/bigquery/service.rb +96 -39
- data/lib/google/cloud/bigquery/standard_sql.rb +257 -53
- data/lib/google/cloud/bigquery/table.rb +417 -67
- data/lib/google/cloud/bigquery/table/async_inserter.rb +18 -8
- data/lib/google/cloud/bigquery/table/list.rb +1 -2
- data/lib/google/cloud/bigquery/time.rb +6 -0
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +10 -7
|
@@ -71,8 +71,7 @@ module Google
|
|
|
71
71
|
def next
|
|
72
72
|
return nil unless next?
|
|
73
73
|
ensure_service!
|
|
74
|
-
|
|
75
|
-
gapi = @service.list_datasets options
|
|
74
|
+
gapi = @service.list_datasets all: @hidden, filter: @filter, token: token, max: @max
|
|
76
75
|
self.class.from_gapi gapi, @service, @hidden, @filter, @max
|
|
77
76
|
end
|
|
78
77
|
|
|
@@ -45,9 +45,12 @@ module Google
|
|
|
45
45
|
# data = bigquery.query "SELECT * FROM my_ext_table",
|
|
46
46
|
# external: { my_ext_table: csv_table }
|
|
47
47
|
#
|
|
48
|
+
# # Iterate over the first page of results
|
|
48
49
|
# data.each do |row|
|
|
49
50
|
# puts row[:name]
|
|
50
51
|
# end
|
|
52
|
+
# # Retrieve the next page of results
|
|
53
|
+
# data = data.next if data.next?
|
|
51
54
|
#
|
|
52
55
|
module External
|
|
53
56
|
##
|
|
@@ -138,9 +141,12 @@ module Google
|
|
|
138
141
|
# data = bigquery.query "SELECT * FROM my_ext_table",
|
|
139
142
|
# external: { my_ext_table: avro_table }
|
|
140
143
|
#
|
|
144
|
+
# # Iterate over the first page of results
|
|
141
145
|
# data.each do |row|
|
|
142
146
|
# puts row[:name]
|
|
143
147
|
# end
|
|
148
|
+
# # Retrieve the next page of results
|
|
149
|
+
# data = data.next if data.next?
|
|
144
150
|
#
|
|
145
151
|
class DataSource
|
|
146
152
|
##
|
|
@@ -575,9 +581,12 @@ module Google
|
|
|
575
581
|
# data = bigquery.query "SELECT * FROM my_ext_table",
|
|
576
582
|
# external: { my_ext_table: csv_table }
|
|
577
583
|
#
|
|
584
|
+
# # Iterate over the first page of results
|
|
578
585
|
# data.each do |row|
|
|
579
586
|
# puts row[:name]
|
|
580
587
|
# end
|
|
588
|
+
# # Retrieve the next page of results
|
|
589
|
+
# data = data.next if data.next?
|
|
581
590
|
#
|
|
582
591
|
class CsvSource < External::DataSource
|
|
583
592
|
##
|
|
@@ -1037,9 +1046,12 @@ module Google
|
|
|
1037
1046
|
# data = bigquery.query "SELECT * FROM my_ext_table",
|
|
1038
1047
|
# external: { my_ext_table: json_table }
|
|
1039
1048
|
#
|
|
1049
|
+
# # Iterate over the first page of results
|
|
1040
1050
|
# data.each do |row|
|
|
1041
1051
|
# puts row[:name]
|
|
1042
1052
|
# end
|
|
1053
|
+
# # Retrieve the next page of results
|
|
1054
|
+
# data = data.next if data.next?
|
|
1043
1055
|
#
|
|
1044
1056
|
class JsonSource < External::DataSource
|
|
1045
1057
|
##
|
|
@@ -1173,9 +1185,12 @@ module Google
|
|
|
1173
1185
|
# data = bigquery.query "SELECT * FROM my_ext_table",
|
|
1174
1186
|
# external: { my_ext_table: sheets_table }
|
|
1175
1187
|
#
|
|
1188
|
+
# # Iterate over the first page of results
|
|
1176
1189
|
# data.each do |row|
|
|
1177
1190
|
# puts row[:name]
|
|
1178
1191
|
# end
|
|
1192
|
+
# # Retrieve the next page of results
|
|
1193
|
+
# data = data.next if data.next?
|
|
1179
1194
|
#
|
|
1180
1195
|
class SheetsSource < External::DataSource
|
|
1181
1196
|
##
|
|
@@ -1318,9 +1333,12 @@ module Google
|
|
|
1318
1333
|
# data = bigquery.query "SELECT * FROM my_ext_table",
|
|
1319
1334
|
# external: { my_ext_table: bigtable_table }
|
|
1320
1335
|
#
|
|
1336
|
+
# # Iterate over the first page of results
|
|
1321
1337
|
# data.each do |row|
|
|
1322
1338
|
# puts row[:name]
|
|
1323
1339
|
# end
|
|
1340
|
+
# # Retrieve the next page of results
|
|
1341
|
+
# data = data.next if data.next?
|
|
1324
1342
|
#
|
|
1325
1343
|
class BigtableSource < External::DataSource
|
|
1326
1344
|
##
|
|
@@ -1516,9 +1534,12 @@ module Google
|
|
|
1516
1534
|
# data = bigquery.query "SELECT * FROM my_ext_table",
|
|
1517
1535
|
# external: { my_ext_table: bigtable_table }
|
|
1518
1536
|
#
|
|
1537
|
+
# # Iterate over the first page of results
|
|
1519
1538
|
# data.each do |row|
|
|
1520
1539
|
# puts row[:name]
|
|
1521
1540
|
# end
|
|
1541
|
+
# # Retrieve the next page of results
|
|
1542
|
+
# data = data.next if data.next?
|
|
1522
1543
|
#
|
|
1523
1544
|
class ColumnFamily
|
|
1524
1545
|
##
|
|
@@ -2053,9 +2074,12 @@ module Google
|
|
|
2053
2074
|
# data = bigquery.query "SELECT * FROM my_ext_table",
|
|
2054
2075
|
# external: { my_ext_table: bigtable_table }
|
|
2055
2076
|
#
|
|
2077
|
+
# # Iterate over the first page of results
|
|
2056
2078
|
# data.each do |row|
|
|
2057
2079
|
# puts row[:name]
|
|
2058
2080
|
# end
|
|
2081
|
+
# # Retrieve the next page of results
|
|
2082
|
+
# data = data.next if data.next?
|
|
2059
2083
|
#
|
|
2060
2084
|
class Column
|
|
2061
2085
|
##
|
|
@@ -182,7 +182,7 @@ module Google
|
|
|
182
182
|
#
|
|
183
183
|
# @return [Google::Cloud::Bigquery::ExtractJob::Updater] A job
|
|
184
184
|
# configuration object for setting query options.
|
|
185
|
-
def self.from_options service, table, storage_files, options
|
|
185
|
+
def self.from_options service, table, storage_files, options
|
|
186
186
|
job_ref = service.job_ref_from options[:job_id], options[:prefix]
|
|
187
187
|
storage_urls = Array(storage_files).map do |url|
|
|
188
188
|
url.respond_to?(:to_gs_url) ? url.to_gs_url : url
|
|
@@ -207,7 +207,7 @@ module Google
|
|
|
207
207
|
#
|
|
208
208
|
# @return [Google::Cloud::Bigquery::ExtractJob::Updater] A job
|
|
209
209
|
# configuration object for setting query options.
|
|
210
|
-
def self.from_job_and_options request, options
|
|
210
|
+
def self.from_job_and_options request, options
|
|
211
211
|
updater = ExtractJob::Updater.new request
|
|
212
212
|
updater.compression = options[:compression]
|
|
213
213
|
updater.delimiter = options[:delimiter]
|
|
@@ -336,6 +336,23 @@ module Google
|
|
|
336
336
|
@gapi.configuration.extract.use_avro_logical_types = value
|
|
337
337
|
end
|
|
338
338
|
|
|
339
|
+
def cancel
|
|
340
|
+
raise "not implemented in #{self.class}"
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
def rerun!
|
|
344
|
+
raise "not implemented in #{self.class}"
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
def reload!
|
|
348
|
+
raise "not implemented in #{self.class}"
|
|
349
|
+
end
|
|
350
|
+
alias refresh! reload!
|
|
351
|
+
|
|
352
|
+
def wait_until_done!
|
|
353
|
+
raise "not implemented in #{self.class}"
|
|
354
|
+
end
|
|
355
|
+
|
|
339
356
|
##
|
|
340
357
|
# @private Returns the Google API client library version of this job.
|
|
341
358
|
#
|
|
@@ -197,6 +197,72 @@ module Google
|
|
|
197
197
|
Convert.millis_to_time @gapi.statistics.end_time
|
|
198
198
|
end
|
|
199
199
|
|
|
200
|
+
##
|
|
201
|
+
# The number of child jobs executed.
|
|
202
|
+
#
|
|
203
|
+
# @return [Integer] The number of child jobs executed.
|
|
204
|
+
#
|
|
205
|
+
def num_child_jobs
|
|
206
|
+
@gapi.statistics.num_child_jobs || 0
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
##
|
|
210
|
+
# If this is a child job, the id of the parent.
|
|
211
|
+
#
|
|
212
|
+
# @return [String, nil] The ID of the parent job, or `nil` if not a child job.
|
|
213
|
+
#
|
|
214
|
+
def parent_job_id
|
|
215
|
+
@gapi.statistics.parent_job_id
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
##
|
|
219
|
+
# The statistics including stack frames for a child job of a script.
|
|
220
|
+
#
|
|
221
|
+
# @return [Google::Cloud::Bigquery::Job::ScriptStatistics, nil] The script statistics, or `nil` if the job is
|
|
222
|
+
# not a child job.
|
|
223
|
+
#
|
|
224
|
+
# @example
|
|
225
|
+
# require "google/cloud/bigquery"
|
|
226
|
+
#
|
|
227
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
228
|
+
#
|
|
229
|
+
# multi_statement_sql = <<~SQL
|
|
230
|
+
# -- Declare a variable to hold names as an array.
|
|
231
|
+
# DECLARE top_names ARRAY<STRING>;
|
|
232
|
+
# -- Build an array of the top 100 names from the year 2017.
|
|
233
|
+
# SET top_names = (
|
|
234
|
+
# SELECT ARRAY_AGG(name ORDER BY number DESC LIMIT 100)
|
|
235
|
+
# FROM `bigquery-public-data.usa_names.usa_1910_current`
|
|
236
|
+
# WHERE year = 2017
|
|
237
|
+
# );
|
|
238
|
+
# -- Which names appear as words in Shakespeare's plays?
|
|
239
|
+
# SELECT
|
|
240
|
+
# name AS shakespeare_name
|
|
241
|
+
# FROM UNNEST(top_names) AS name
|
|
242
|
+
# WHERE name IN (
|
|
243
|
+
# SELECT word
|
|
244
|
+
# FROM `bigquery-public-data.samples.shakespeare`
|
|
245
|
+
# );
|
|
246
|
+
# SQL
|
|
247
|
+
#
|
|
248
|
+
# job = bigquery.query_job multi_statement_sql
|
|
249
|
+
#
|
|
250
|
+
# job.wait_until_done!
|
|
251
|
+
#
|
|
252
|
+
# child_jobs = bigquery.jobs parent_job: job
|
|
253
|
+
#
|
|
254
|
+
# child_jobs.each do |child_job|
|
|
255
|
+
# script_statistics = child_job.script_statistics
|
|
256
|
+
# puts script_statistics.evaluation_kind
|
|
257
|
+
# script_statistics.stack_frames.each do |stack_frame|
|
|
258
|
+
# puts stack_frame.text
|
|
259
|
+
# end
|
|
260
|
+
# end
|
|
261
|
+
#
|
|
262
|
+
def script_statistics
|
|
263
|
+
ScriptStatistics.from_gapi @gapi.statistics.script_statistics if @gapi.statistics.script_statistics
|
|
264
|
+
end
|
|
265
|
+
|
|
200
266
|
##
|
|
201
267
|
# The configuration for the job. Returns a hash.
|
|
202
268
|
#
|
|
@@ -423,6 +489,138 @@ module Google
|
|
|
423
489
|
end
|
|
424
490
|
end
|
|
425
491
|
|
|
492
|
+
##
|
|
493
|
+
# Represents statistics for a child job of a script.
|
|
494
|
+
#
|
|
495
|
+
# @attr_reader [String] evaluation_kind Indicates the type of child job. Possible values include `STATEMENT` and
|
|
496
|
+
# `EXPRESSION`.
|
|
497
|
+
# @attr_reader [Array<Google::Cloud::Bigquery::Job::ScriptStackFrame>] stack_frames Stack trace where the
|
|
498
|
+
# current evaluation happened. Shows line/column/procedure name of each frame on the stack at the point where
|
|
499
|
+
# the current evaluation happened. The leaf frame is first, the primary script is last.
|
|
500
|
+
#
|
|
501
|
+
# @example
|
|
502
|
+
# require "google/cloud/bigquery"
|
|
503
|
+
#
|
|
504
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
505
|
+
#
|
|
506
|
+
# multi_statement_sql = <<~SQL
|
|
507
|
+
# -- Declare a variable to hold names as an array.
|
|
508
|
+
# DECLARE top_names ARRAY<STRING>;
|
|
509
|
+
# -- Build an array of the top 100 names from the year 2017.
|
|
510
|
+
# SET top_names = (
|
|
511
|
+
# SELECT ARRAY_AGG(name ORDER BY number DESC LIMIT 100)
|
|
512
|
+
# FROM `bigquery-public-data.usa_names.usa_1910_current`
|
|
513
|
+
# WHERE year = 2017
|
|
514
|
+
# );
|
|
515
|
+
# -- Which names appear as words in Shakespeare's plays?
|
|
516
|
+
# SELECT
|
|
517
|
+
# name AS shakespeare_name
|
|
518
|
+
# FROM UNNEST(top_names) AS name
|
|
519
|
+
# WHERE name IN (
|
|
520
|
+
# SELECT word
|
|
521
|
+
# FROM `bigquery-public-data.samples.shakespeare`
|
|
522
|
+
# );
|
|
523
|
+
# SQL
|
|
524
|
+
#
|
|
525
|
+
# job = bigquery.query_job multi_statement_sql
|
|
526
|
+
#
|
|
527
|
+
# job.wait_until_done!
|
|
528
|
+
#
|
|
529
|
+
# child_jobs = bigquery.jobs parent_job: job
|
|
530
|
+
#
|
|
531
|
+
# child_jobs.each do |child_job|
|
|
532
|
+
# script_statistics = child_job.script_statistics
|
|
533
|
+
# puts script_statistics.evaluation_kind
|
|
534
|
+
# script_statistics.stack_frames.each do |stack_frame|
|
|
535
|
+
# puts stack_frame.text
|
|
536
|
+
# end
|
|
537
|
+
# end
|
|
538
|
+
#
|
|
539
|
+
class ScriptStatistics
|
|
540
|
+
attr_reader :evaluation_kind, :stack_frames
|
|
541
|
+
|
|
542
|
+
##
|
|
543
|
+
# @private Creates a new ScriptStatistics instance.
|
|
544
|
+
def initialize evaluation_kind, stack_frames
|
|
545
|
+
@evaluation_kind = evaluation_kind
|
|
546
|
+
@stack_frames = stack_frames
|
|
547
|
+
end
|
|
548
|
+
|
|
549
|
+
##
|
|
550
|
+
# @private New ScriptStatistics from a statistics.script_statistics object.
|
|
551
|
+
def self.from_gapi gapi
|
|
552
|
+
frames = Array(gapi.stack_frames).map { |g| ScriptStackFrame.from_gapi g }
|
|
553
|
+
new gapi.evaluation_kind, frames
|
|
554
|
+
end
|
|
555
|
+
end
|
|
556
|
+
|
|
557
|
+
##
|
|
558
|
+
# Represents a stack frame showing the line/column/procedure name where the current evaluation happened.
|
|
559
|
+
#
|
|
560
|
+
# @attr_reader [Integer] start_line One-based start line.
|
|
561
|
+
# @attr_reader [Integer] start_column One-based start column.
|
|
562
|
+
# @attr_reader [Integer] end_line One-based end line.
|
|
563
|
+
# @attr_reader [Integer] end_column One-based end column.
|
|
564
|
+
# @attr_reader [String] text Text of the current statement/expression.
|
|
565
|
+
#
|
|
566
|
+
# @example
|
|
567
|
+
# require "google/cloud/bigquery"
|
|
568
|
+
#
|
|
569
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
570
|
+
#
|
|
571
|
+
# multi_statement_sql = <<~SQL
|
|
572
|
+
# -- Declare a variable to hold names as an array.
|
|
573
|
+
# DECLARE top_names ARRAY<STRING>;
|
|
574
|
+
# -- Build an array of the top 100 names from the year 2017.
|
|
575
|
+
# SET top_names = (
|
|
576
|
+
# SELECT ARRAY_AGG(name ORDER BY number DESC LIMIT 100)
|
|
577
|
+
# FROM `bigquery-public-data.usa_names.usa_1910_current`
|
|
578
|
+
# WHERE year = 2017
|
|
579
|
+
# );
|
|
580
|
+
# -- Which names appear as words in Shakespeare's plays?
|
|
581
|
+
# SELECT
|
|
582
|
+
# name AS shakespeare_name
|
|
583
|
+
# FROM UNNEST(top_names) AS name
|
|
584
|
+
# WHERE name IN (
|
|
585
|
+
# SELECT word
|
|
586
|
+
# FROM `bigquery-public-data.samples.shakespeare`
|
|
587
|
+
# );
|
|
588
|
+
# SQL
|
|
589
|
+
#
|
|
590
|
+
# job = bigquery.query_job multi_statement_sql
|
|
591
|
+
#
|
|
592
|
+
# job.wait_until_done!
|
|
593
|
+
#
|
|
594
|
+
# child_jobs = bigquery.jobs parent_job: job
|
|
595
|
+
#
|
|
596
|
+
# child_jobs.each do |child_job|
|
|
597
|
+
# script_statistics = child_job.script_statistics
|
|
598
|
+
# puts script_statistics.evaluation_kind
|
|
599
|
+
# script_statistics.stack_frames.each do |stack_frame|
|
|
600
|
+
# puts stack_frame.text
|
|
601
|
+
# end
|
|
602
|
+
# end
|
|
603
|
+
#
|
|
604
|
+
class ScriptStackFrame
|
|
605
|
+
attr_reader :start_line, :start_column, :end_line, :end_column, :text
|
|
606
|
+
|
|
607
|
+
##
|
|
608
|
+
# @private Creates a new ScriptStackFrame instance.
|
|
609
|
+
def initialize start_line, start_column, end_line, end_column, text
|
|
610
|
+
@start_line = start_line
|
|
611
|
+
@start_column = start_column
|
|
612
|
+
@end_line = end_line
|
|
613
|
+
@end_column = end_column
|
|
614
|
+
@text = text
|
|
615
|
+
end
|
|
616
|
+
|
|
617
|
+
##
|
|
618
|
+
# @private New ScriptStackFrame from a statistics.script_statistics[].stack_frames element.
|
|
619
|
+
def self.from_gapi gapi
|
|
620
|
+
new gapi.start_line, gapi.start_column, gapi.end_line, gapi.end_column, gapi.text
|
|
621
|
+
end
|
|
622
|
+
end
|
|
623
|
+
|
|
426
624
|
protected
|
|
427
625
|
|
|
428
626
|
##
|
|
@@ -71,9 +71,9 @@ module Google
|
|
|
71
71
|
def next
|
|
72
72
|
return nil unless next?
|
|
73
73
|
ensure_service!
|
|
74
|
-
|
|
75
|
-
next_gapi = @service.list_jobs
|
|
76
|
-
self.class.from_gapi next_gapi, @service,
|
|
74
|
+
next_kwargs = @kwargs.merge token: token
|
|
75
|
+
next_gapi = @service.list_jobs next_kwargs
|
|
76
|
+
self.class.from_gapi next_gapi, @service, next_kwargs
|
|
77
77
|
end
|
|
78
78
|
|
|
79
79
|
##
|
|
@@ -139,12 +139,12 @@ module Google
|
|
|
139
139
|
##
|
|
140
140
|
# @private New Job::List from a Google API Client
|
|
141
141
|
# Google::Apis::BigqueryV2::JobList object.
|
|
142
|
-
def self.from_gapi gapi_list, service,
|
|
142
|
+
def self.from_gapi gapi_list, service, **kwargs
|
|
143
143
|
jobs = List.new(Array(gapi_list.jobs).map { |gapi_object| Job.from_gapi gapi_object, service })
|
|
144
144
|
jobs.instance_variable_set :@token, gapi_list.next_page_token
|
|
145
145
|
jobs.instance_variable_set :@etag, gapi_list.etag
|
|
146
146
|
jobs.instance_variable_set :@service, service
|
|
147
|
-
jobs.instance_variable_set :@
|
|
147
|
+
jobs.instance_variable_set :@kwargs, kwargs
|
|
148
148
|
jobs
|
|
149
149
|
end
|
|
150
150
|
|
|
@@ -348,7 +348,71 @@ module Google
|
|
|
348
348
|
end
|
|
349
349
|
|
|
350
350
|
###
|
|
351
|
-
# Checks if the destination table will be
|
|
351
|
+
# Checks if the destination table will be range partitioned. See [Creating and using integer range partitioned
|
|
352
|
+
# tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
|
|
353
|
+
#
|
|
354
|
+
# @return [Boolean] `true` when the table is range partitioned, or `false` otherwise.
|
|
355
|
+
#
|
|
356
|
+
# @!group Attributes
|
|
357
|
+
#
|
|
358
|
+
def range_partitioning?
|
|
359
|
+
!@gapi.configuration.load.range_partitioning.nil?
|
|
360
|
+
end
|
|
361
|
+
|
|
362
|
+
###
|
|
363
|
+
# The field on which the destination table will be range partitioned, if any. The field must be a
|
|
364
|
+
# top-level `NULLABLE/REQUIRED` field. The only supported type is `INTEGER/INT64`. See
|
|
365
|
+
# [Creating and using integer range partitioned
|
|
366
|
+
# tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
|
|
367
|
+
#
|
|
368
|
+
# @return [String, nil] The partition field, if a field was configured, or `nil` if not range partitioned.
|
|
369
|
+
#
|
|
370
|
+
# @!group Attributes
|
|
371
|
+
#
|
|
372
|
+
def range_partitioning_field
|
|
373
|
+
@gapi.configuration.load.range_partitioning.field if range_partitioning?
|
|
374
|
+
end
|
|
375
|
+
|
|
376
|
+
###
|
|
377
|
+
# The start of range partitioning, inclusive. See [Creating and using integer range partitioned
|
|
378
|
+
# tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
|
|
379
|
+
#
|
|
380
|
+
# @return [Integer, nil] The start of range partitioning, inclusive, or `nil` if not range partitioned.
|
|
381
|
+
#
|
|
382
|
+
# @!group Attributes
|
|
383
|
+
#
|
|
384
|
+
def range_partitioning_start
|
|
385
|
+
@gapi.configuration.load.range_partitioning.range.start if range_partitioning?
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
###
|
|
389
|
+
# The width of each interval. See [Creating and using integer range partitioned
|
|
390
|
+
# tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
|
|
391
|
+
#
|
|
392
|
+
# @return [Integer, nil] The width of each interval, for data in range partitions, or `nil` if not range
|
|
393
|
+
# partitioned.
|
|
394
|
+
#
|
|
395
|
+
# @!group Attributes
|
|
396
|
+
#
|
|
397
|
+
def range_partitioning_interval
|
|
398
|
+
return nil unless range_partitioning?
|
|
399
|
+
@gapi.configuration.load.range_partitioning.range.interval
|
|
400
|
+
end
|
|
401
|
+
|
|
402
|
+
###
|
|
403
|
+
# The end of range partitioning, exclusive. See [Creating and using integer range partitioned
|
|
404
|
+
# tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
|
|
405
|
+
#
|
|
406
|
+
# @return [Integer, nil] The end of range partitioning, exclusive, or `nil` if not range partitioned.
|
|
407
|
+
#
|
|
408
|
+
# @!group Attributes
|
|
409
|
+
#
|
|
410
|
+
def range_partitioning_end
|
|
411
|
+
@gapi.configuration.load.range_partitioning.range.end if range_partitioning?
|
|
412
|
+
end
|
|
413
|
+
|
|
414
|
+
###
|
|
415
|
+
# Checks if the destination table will be time partitioned. See
|
|
352
416
|
# [Partitioned Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
|
|
353
417
|
#
|
|
354
418
|
# @return [Boolean, nil] `true` when the table will be time-partitioned,
|
|
@@ -361,10 +425,10 @@ module Google
|
|
|
361
425
|
end
|
|
362
426
|
|
|
363
427
|
###
|
|
364
|
-
# The period for which the destination table will be partitioned, if
|
|
428
|
+
# The period for which the destination table will be time partitioned, if
|
|
365
429
|
# any. See [Partitioned Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
|
|
366
430
|
#
|
|
367
|
-
# @return [String, nil] The partition type. Currently the only supported
|
|
431
|
+
# @return [String, nil] The time partition type. Currently the only supported
|
|
368
432
|
# value is "DAY", or `nil` if not present.
|
|
369
433
|
#
|
|
370
434
|
# @!group Attributes
|
|
@@ -374,13 +438,13 @@ module Google
|
|
|
374
438
|
end
|
|
375
439
|
|
|
376
440
|
###
|
|
377
|
-
# The field on which the destination table will be partitioned, if any.
|
|
378
|
-
# If not set, the destination table will be partitioned by pseudo column
|
|
379
|
-
# `_PARTITIONTIME`; if set, the table will be partitioned by this field.
|
|
441
|
+
# The field on which the destination table will be time partitioned, if any.
|
|
442
|
+
# If not set, the destination table will be time partitioned by pseudo column
|
|
443
|
+
# `_PARTITIONTIME`; if set, the table will be time partitioned by this field.
|
|
380
444
|
# See [Partitioned Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
|
|
381
445
|
#
|
|
382
|
-
# @return [String, nil] The partition field, if a field was configured.
|
|
383
|
-
# `nil` if not partitioned or not set (partitioned by pseudo column
|
|
446
|
+
# @return [String, nil] The time partition field, if a field was configured.
|
|
447
|
+
# `nil` if not time partitioned or not set (partitioned by pseudo column
|
|
384
448
|
# '_PARTITIONTIME').
|
|
385
449
|
#
|
|
386
450
|
# @!group Attributes
|
|
@@ -390,12 +454,12 @@ module Google
|
|
|
390
454
|
end
|
|
391
455
|
|
|
392
456
|
###
|
|
393
|
-
# The expiration for the destination table partitions, if any, in
|
|
457
|
+
# The expiration for the destination table time partitions, if any, in
|
|
394
458
|
# seconds. See [Partitioned
|
|
395
459
|
# Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
|
|
396
460
|
#
|
|
397
461
|
# @return [Integer, nil] The expiration time, in seconds, for data in
|
|
398
|
-
# partitions, or `nil` if not present.
|
|
462
|
+
# time partitions, or `nil` if not present.
|
|
399
463
|
#
|
|
400
464
|
# @!group Attributes
|
|
401
465
|
#
|
|
@@ -408,11 +472,11 @@ module Google
|
|
|
408
472
|
|
|
409
473
|
###
|
|
410
474
|
# If set to true, queries over the destination table will require a
|
|
411
|
-
# partition filter that can be used for partition elimination to be
|
|
475
|
+
# time partition filter that can be used for partition elimination to be
|
|
412
476
|
# specified. See [Partitioned
|
|
413
477
|
# Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
|
|
414
478
|
#
|
|
415
|
-
# @return [Boolean] `true` when a partition filter will be required,
|
|
479
|
+
# @return [Boolean] `true` when a time partition filter will be required,
|
|
416
480
|
# or `false` otherwise.
|
|
417
481
|
#
|
|
418
482
|
# @!group Attributes
|
|
@@ -1253,14 +1317,180 @@ module Google
|
|
|
1253
1317
|
end
|
|
1254
1318
|
|
|
1255
1319
|
##
|
|
1256
|
-
# Sets the
|
|
1320
|
+
# Sets the field on which to range partition the table. See [Creating and using integer range partitioned
|
|
1321
|
+
# tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
|
|
1322
|
+
#
|
|
1323
|
+
# See {#range_partitioning_start=}, {#range_partitioning_interval=} and {#range_partitioning_end=}.
|
|
1324
|
+
#
|
|
1325
|
+
# You can only set range partitioning when creating a table. BigQuery does not allow you to change
|
|
1326
|
+
# partitioning on an existing table.
|
|
1327
|
+
#
|
|
1328
|
+
# @param [String] field The range partition field. the destination table is partitioned by this
|
|
1329
|
+
# field. The field must be a top-level `NULLABLE/REQUIRED` field. The only supported
|
|
1330
|
+
# type is `INTEGER/INT64`.
|
|
1331
|
+
#
|
|
1332
|
+
# @example
|
|
1333
|
+
# require "google/cloud/bigquery"
|
|
1334
|
+
#
|
|
1335
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
1336
|
+
# dataset = bigquery.dataset "my_dataset"
|
|
1337
|
+
#
|
|
1338
|
+
# gs_url = "gs://my-bucket/file-name.csv"
|
|
1339
|
+
# load_job = dataset.load_job "my_new_table", gs_url do |job|
|
|
1340
|
+
# job.schema do |schema|
|
|
1341
|
+
# schema.integer "my_table_id", mode: :required
|
|
1342
|
+
# schema.string "my_table_data", mode: :required
|
|
1343
|
+
# end
|
|
1344
|
+
# job.range_partitioning_field = "my_table_id"
|
|
1345
|
+
# job.range_partitioning_start = 0
|
|
1346
|
+
# job.range_partitioning_interval = 10
|
|
1347
|
+
# job.range_partitioning_end = 100
|
|
1348
|
+
# end
|
|
1349
|
+
#
|
|
1350
|
+
# load_job.wait_until_done!
|
|
1351
|
+
# load_job.done? #=> true
|
|
1352
|
+
#
|
|
1353
|
+
# @!group Attributes
|
|
1354
|
+
#
|
|
1355
|
+
def range_partitioning_field= field
|
|
1356
|
+
@gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
|
|
1357
|
+
range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
|
|
1358
|
+
)
|
|
1359
|
+
@gapi.configuration.load.range_partitioning.field = field
|
|
1360
|
+
end
|
|
1361
|
+
|
|
1362
|
+
##
|
|
1363
|
+
# Sets the start of range partitioning, inclusive, for the destination table. See [Creating and using integer
|
|
1364
|
+
# range partitioned tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
|
|
1365
|
+
#
|
|
1366
|
+
# You can only set range partitioning when creating a table. BigQuery does not allow you to change
|
|
1367
|
+
# partitioning on an existing table.
|
|
1368
|
+
#
|
|
1369
|
+
# See {#range_partitioning_field=}, {#range_partitioning_interval=} and {#range_partitioning_end=}.
|
|
1370
|
+
#
|
|
1371
|
+
# @param [Integer] range_start The start of range partitioning, inclusive.
|
|
1372
|
+
#
|
|
1373
|
+
# @example
|
|
1374
|
+
# require "google/cloud/bigquery"
|
|
1375
|
+
#
|
|
1376
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
1377
|
+
# dataset = bigquery.dataset "my_dataset"
|
|
1378
|
+
#
|
|
1379
|
+
# gs_url = "gs://my-bucket/file-name.csv"
|
|
1380
|
+
# load_job = dataset.load_job "my_new_table", gs_url do |job|
|
|
1381
|
+
# job.schema do |schema|
|
|
1382
|
+
# schema.integer "my_table_id", mode: :required
|
|
1383
|
+
# schema.string "my_table_data", mode: :required
|
|
1384
|
+
# end
|
|
1385
|
+
# job.range_partitioning_field = "my_table_id"
|
|
1386
|
+
# job.range_partitioning_start = 0
|
|
1387
|
+
# job.range_partitioning_interval = 10
|
|
1388
|
+
# job.range_partitioning_end = 100
|
|
1389
|
+
# end
|
|
1390
|
+
#
|
|
1391
|
+
# load_job.wait_until_done!
|
|
1392
|
+
# load_job.done? #=> true
|
|
1393
|
+
#
|
|
1394
|
+
# @!group Attributes
|
|
1395
|
+
#
|
|
1396
|
+
def range_partitioning_start= range_start
|
|
1397
|
+
@gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
|
|
1398
|
+
range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
|
|
1399
|
+
)
|
|
1400
|
+
@gapi.configuration.load.range_partitioning.range.start = range_start
|
|
1401
|
+
end
|
|
1402
|
+
|
|
1403
|
+
##
|
|
1404
|
+
# Sets width of each interval for data in range partitions. See [Creating and using integer range partitioned
|
|
1405
|
+
# tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
|
|
1406
|
+
#
|
|
1407
|
+
# You can only set range partitioning when creating a table. BigQuery does not allow you to change
|
|
1408
|
+
# partitioning on an existing table.
|
|
1409
|
+
#
|
|
1410
|
+
# See {#range_partitioning_field=}, {#range_partitioning_start=} and {#range_partitioning_end=}.
|
|
1411
|
+
#
|
|
1412
|
+
# @param [Integer] range_interval The width of each interval, for data in partitions.
|
|
1413
|
+
#
|
|
1414
|
+
# @example
|
|
1415
|
+
# require "google/cloud/bigquery"
|
|
1416
|
+
#
|
|
1417
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
1418
|
+
# dataset = bigquery.dataset "my_dataset"
|
|
1419
|
+
#
|
|
1420
|
+
# gs_url = "gs://my-bucket/file-name.csv"
|
|
1421
|
+
# load_job = dataset.load_job "my_new_table", gs_url do |job|
|
|
1422
|
+
# job.schema do |schema|
|
|
1423
|
+
# schema.integer "my_table_id", mode: :required
|
|
1424
|
+
# schema.string "my_table_data", mode: :required
|
|
1425
|
+
# end
|
|
1426
|
+
# job.range_partitioning_field = "my_table_id"
|
|
1427
|
+
# job.range_partitioning_start = 0
|
|
1428
|
+
# job.range_partitioning_interval = 10
|
|
1429
|
+
# job.range_partitioning_end = 100
|
|
1430
|
+
# end
|
|
1431
|
+
#
|
|
1432
|
+
# load_job.wait_until_done!
|
|
1433
|
+
# load_job.done? #=> true
|
|
1434
|
+
#
|
|
1435
|
+
# @!group Attributes
|
|
1436
|
+
#
|
|
1437
|
+
def range_partitioning_interval= range_interval
|
|
1438
|
+
@gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
|
|
1439
|
+
range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
|
|
1440
|
+
)
|
|
1441
|
+
@gapi.configuration.load.range_partitioning.range.interval = range_interval
|
|
1442
|
+
end
|
|
1443
|
+
|
|
1444
|
+
##
|
|
1445
|
+
# Sets the end of range partitioning, exclusive, for the destination table. See [Creating and using integer
|
|
1446
|
+
# range partitioned tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
|
|
1447
|
+
#
|
|
1448
|
+
# You can only set range partitioning when creating a table. BigQuery does not allow you to change
|
|
1449
|
+
# partitioning on an existing table.
|
|
1450
|
+
#
|
|
1451
|
+
# See {#range_partitioning_start=}, {#range_partitioning_interval=} and {#range_partitioning_field=}.
|
|
1452
|
+
#
|
|
1453
|
+
# @param [Integer] range_end The end of range partitioning, exclusive.
|
|
1454
|
+
#
|
|
1455
|
+
# @example
|
|
1456
|
+
# require "google/cloud/bigquery"
|
|
1457
|
+
#
|
|
1458
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
1459
|
+
# dataset = bigquery.dataset "my_dataset"
|
|
1460
|
+
#
|
|
1461
|
+
# gs_url = "gs://my-bucket/file-name.csv"
|
|
1462
|
+
# load_job = dataset.load_job "my_new_table", gs_url do |job|
|
|
1463
|
+
# job.schema do |schema|
|
|
1464
|
+
# schema.integer "my_table_id", mode: :required
|
|
1465
|
+
# schema.string "my_table_data", mode: :required
|
|
1466
|
+
# end
|
|
1467
|
+
# job.range_partitioning_field = "my_table_id"
|
|
1468
|
+
# job.range_partitioning_start = 0
|
|
1469
|
+
# job.range_partitioning_interval = 10
|
|
1470
|
+
# job.range_partitioning_end = 100
|
|
1471
|
+
# end
|
|
1472
|
+
#
|
|
1473
|
+
# load_job.wait_until_done!
|
|
1474
|
+
# load_job.done? #=> true
|
|
1475
|
+
#
|
|
1476
|
+
# @!group Attributes
|
|
1477
|
+
#
|
|
1478
|
+
def range_partitioning_end= range_end
|
|
1479
|
+
@gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
|
|
1480
|
+
range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
|
|
1481
|
+
)
|
|
1482
|
+
@gapi.configuration.load.range_partitioning.range.end = range_end
|
|
1483
|
+
end
|
|
1484
|
+
|
|
1485
|
+
##
|
|
1486
|
+
# Sets the time partitioning for the destination table. See [Partitioned
|
|
1257
1487
|
# Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
|
|
1258
1488
|
#
|
|
1259
|
-
# You can only set the partitioning field while creating a table.
|
|
1489
|
+
# You can only set the time partitioning field while creating a table.
|
|
1260
1490
|
# BigQuery does not allow you to change partitioning on an existing
|
|
1261
1491
|
# table.
|
|
1262
1492
|
#
|
|
1263
|
-
# @param [String] type The partition type. Currently the only
|
|
1493
|
+
# @param [String] type The time partition type. Currently the only
|
|
1264
1494
|
# supported value is "DAY".
|
|
1265
1495
|
#
|
|
1266
1496
|
# @example
|
|
@@ -1285,20 +1515,20 @@ module Google
|
|
|
1285
1515
|
end
|
|
1286
1516
|
|
|
1287
1517
|
##
|
|
1288
|
-
# Sets the field on which to partition the destination table. If not
|
|
1289
|
-
# set, the destination table is partitioned by pseudo column
|
|
1290
|
-
# `_PARTITIONTIME`; if set, the table is partitioned by this field.
|
|
1518
|
+
# Sets the field on which to time partition the destination table. If not
|
|
1519
|
+
# set, the destination table is time partitioned by pseudo column
|
|
1520
|
+
# `_PARTITIONTIME`; if set, the table is time partitioned by this field.
|
|
1291
1521
|
# See [Partitioned
|
|
1292
1522
|
# Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
|
|
1293
1523
|
#
|
|
1294
|
-
# The destination table must also be partitioned. See
|
|
1524
|
+
# The destination table must also be time partitioned. See
|
|
1295
1525
|
# {#time_partitioning_type=}.
|
|
1296
1526
|
#
|
|
1297
|
-
# You can only set the partitioning field while creating a table.
|
|
1527
|
+
# You can only set the time partitioning field while creating a table.
|
|
1298
1528
|
# BigQuery does not allow you to change partitioning on an existing
|
|
1299
1529
|
# table.
|
|
1300
1530
|
#
|
|
1301
|
-
# @param [String] field The partition field. The field must be a
|
|
1531
|
+
# @param [String] field The time partition field. The field must be a
|
|
1302
1532
|
# top-level TIMESTAMP or DATE field. Its mode must be NULLABLE or
|
|
1303
1533
|
# REQUIRED.
|
|
1304
1534
|
#
|
|
@@ -1328,15 +1558,15 @@ module Google
|
|
|
1328
1558
|
end
|
|
1329
1559
|
|
|
1330
1560
|
##
|
|
1331
|
-
# Sets the partition expiration for the destination table. See
|
|
1561
|
+
# Sets the time partition expiration for the destination table. See
|
|
1332
1562
|
# [Partitioned
|
|
1333
1563
|
# Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
|
|
1334
1564
|
#
|
|
1335
|
-
# The destination table must also be partitioned. See
|
|
1565
|
+
# The destination table must also be time partitioned. See
|
|
1336
1566
|
# {#time_partitioning_type=}.
|
|
1337
1567
|
#
|
|
1338
1568
|
# @param [Integer] expiration An expiration time, in seconds,
|
|
1339
|
-
# for data in partitions.
|
|
1569
|
+
# for data in time partitions.
|
|
1340
1570
|
#
|
|
1341
1571
|
# @example
|
|
1342
1572
|
# require "google/cloud/bigquery"
|
|
@@ -1362,12 +1592,12 @@ module Google
|
|
|
1362
1592
|
|
|
1363
1593
|
##
|
|
1364
1594
|
# If set to true, queries over the destination table will require a
|
|
1365
|
-
# partition filter that can be used for partition elimination to be
|
|
1595
|
+
# time partition filter that can be used for time partition elimination to be
|
|
1366
1596
|
# specified. See [Partitioned
|
|
1367
1597
|
# Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
|
|
1368
1598
|
#
|
|
1369
1599
|
# @param [Boolean] val Indicates if queries over the destination table
|
|
1370
|
-
# will require a partition filter. The default value is `false`.
|
|
1600
|
+
# will require a time partition filter. The default value is `false`.
|
|
1371
1601
|
#
|
|
1372
1602
|
# @!group Attributes
|
|
1373
1603
|
#
|
|
@@ -1426,6 +1656,23 @@ module Google
|
|
|
1426
1656
|
@gapi.configuration.load.clustering.fields = fields
|
|
1427
1657
|
end
|
|
1428
1658
|
|
|
1659
|
+
def cancel
|
|
1660
|
+
raise "not implemented in #{self.class}"
|
|
1661
|
+
end
|
|
1662
|
+
|
|
1663
|
+
def rerun!
|
|
1664
|
+
raise "not implemented in #{self.class}"
|
|
1665
|
+
end
|
|
1666
|
+
|
|
1667
|
+
def reload!
|
|
1668
|
+
raise "not implemented in #{self.class}"
|
|
1669
|
+
end
|
|
1670
|
+
alias refresh! reload!
|
|
1671
|
+
|
|
1672
|
+
def wait_until_done!
|
|
1673
|
+
raise "not implemented in #{self.class}"
|
|
1674
|
+
end
|
|
1675
|
+
|
|
1429
1676
|
##
|
|
1430
1677
|
# @private Returns the Google API client library version of this job.
|
|
1431
1678
|
#
|