google-cloud-bigquery 1.20.0 → 1.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +39 -0
- data/TROUBLESHOOTING.md +2 -8
- data/lib/google-cloud-bigquery.rb +9 -2
- data/lib/google/cloud/bigquery.rb +1 -1
- data/lib/google/cloud/bigquery/convert.rb +3 -1
- data/lib/google/cloud/bigquery/copy_job.rb +15 -6
- data/lib/google/cloud/bigquery/data.rb +12 -0
- data/lib/google/cloud/bigquery/dataset.rb +85 -28
- data/lib/google/cloud/bigquery/external.rb +24 -0
- data/lib/google/cloud/bigquery/extract_job.rb +153 -45
- data/lib/google/cloud/bigquery/job.rb +198 -0
- data/lib/google/cloud/bigquery/load_job.rb +15 -6
- data/lib/google/cloud/bigquery/model.rb +164 -8
- data/lib/google/cloud/bigquery/project.rb +242 -84
- data/lib/google/cloud/bigquery/query_job.rb +56 -6
- data/lib/google/cloud/bigquery/service.rb +19 -13
- data/lib/google/cloud/bigquery/table.rb +82 -41
- data/lib/google/cloud/bigquery/time.rb +6 -0
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +5 -5
@@ -48,6 +48,44 @@ module Google
|
|
48
48
|
# puts job.data.first
|
49
49
|
# end
|
50
50
|
#
|
51
|
+
# @example With multiple statements and child jobs:
|
52
|
+
# require "google/cloud/bigquery"
|
53
|
+
#
|
54
|
+
# bigquery = Google::Cloud::Bigquery.new
|
55
|
+
#
|
56
|
+
# multi_statement_sql = <<~SQL
|
57
|
+
# -- Declare a variable to hold names as an array.
|
58
|
+
# DECLARE top_names ARRAY<STRING>;
|
59
|
+
# -- Build an array of the top 100 names from the year 2017.
|
60
|
+
# SET top_names = (
|
61
|
+
# SELECT ARRAY_AGG(name ORDER BY number DESC LIMIT 100)
|
62
|
+
# FROM `bigquery-public-data.usa_names.usa_1910_current`
|
63
|
+
# WHERE year = 2017
|
64
|
+
# );
|
65
|
+
# -- Which names appear as words in Shakespeare's plays?
|
66
|
+
# SELECT
|
67
|
+
# name AS shakespeare_name
|
68
|
+
# FROM UNNEST(top_names) AS name
|
69
|
+
# WHERE name IN (
|
70
|
+
# SELECT word
|
71
|
+
# FROM `bigquery-public-data.samples.shakespeare`
|
72
|
+
# );
|
73
|
+
# SQL
|
74
|
+
#
|
75
|
+
# job = bigquery.query_job multi_statement_sql
|
76
|
+
#
|
77
|
+
# job.wait_until_done!
|
78
|
+
#
|
79
|
+
# child_jobs = bigquery.jobs parent_job: job
|
80
|
+
#
|
81
|
+
# child_jobs.each do |child_job|
|
82
|
+
# script_statistics = child_job.script_statistics
|
83
|
+
# puts script_statistics.evaluation_kind
|
84
|
+
# script_statistics.stack_frames.each do |stack_frame|
|
85
|
+
# puts stack_frame.text
|
86
|
+
# end
|
87
|
+
# end
|
88
|
+
#
|
51
89
|
class QueryJob < Job
|
52
90
|
##
|
53
91
|
# Checks if the priority for the query is `BATCH`.
|
@@ -637,9 +675,12 @@ module Google
|
|
637
675
|
#
|
638
676
|
# job.wait_until_done!
|
639
677
|
# data = job.data
|
678
|
+
#
|
679
|
+
# # Iterate over the first page of results
|
640
680
|
# data.each do |row|
|
641
681
|
# puts row[:word]
|
642
682
|
# end
|
683
|
+
# # Retrieve the next page of results
|
643
684
|
# data = data.next if data.next?
|
644
685
|
#
|
645
686
|
def data token: nil, max: nil, start: nil
|
@@ -990,12 +1031,21 @@ module Google
|
|
990
1031
|
# Sets the labels to use for the job.
|
991
1032
|
#
|
992
1033
|
# @param [Hash] value A hash of user-provided labels associated with
|
993
|
-
# the job. You can use these to organize and group your jobs.
|
994
|
-
#
|
995
|
-
#
|
996
|
-
#
|
997
|
-
#
|
998
|
-
#
|
1034
|
+
# the job. You can use these to organize and group your jobs.
|
1035
|
+
#
|
1036
|
+
# The labels applied to a resource must meet the following requirements:
|
1037
|
+
#
|
1038
|
+
# * Each resource can have multiple labels, up to a maximum of 64.
|
1039
|
+
# * Each label must be a key-value pair.
|
1040
|
+
# * Keys have a minimum length of 1 character and a maximum length of
|
1041
|
+
# 63 characters, and cannot be empty. Values can be empty, and have
|
1042
|
+
# a maximum length of 63 characters.
|
1043
|
+
# * Keys and values can contain only lowercase letters, numeric characters,
|
1044
|
+
# underscores, and dashes. All characters must use UTF-8 encoding, and
|
1045
|
+
# international characters are allowed.
|
1046
|
+
# * The key portion of a label must be unique. However, you can use the
|
1047
|
+
# same key with multiple resources.
|
1048
|
+
# * Keys must start with a lowercase letter or international character.
|
999
1049
|
#
|
1000
1050
|
# @!group Attributes
|
1001
1051
|
#
|
@@ -43,12 +43,13 @@ module Google
|
|
43
43
|
|
44
44
|
##
|
45
45
|
# Creates a new Service instance.
|
46
|
-
def initialize project, credentials, retries: nil, timeout: nil, host: nil
|
46
|
+
def initialize project, credentials, retries: nil, timeout: nil, host: nil, quota_project: nil
|
47
47
|
@project = project
|
48
48
|
@credentials = credentials
|
49
49
|
@retries = retries
|
50
50
|
@timeout = timeout
|
51
51
|
@host = host
|
52
|
+
@quota_project = quota_project
|
52
53
|
end
|
53
54
|
|
54
55
|
def service
|
@@ -64,6 +65,7 @@ module Google
|
|
64
65
|
service.request_options.header ||= {}
|
65
66
|
service.request_options.header["x-goog-api-client"] = \
|
66
67
|
"gl-ruby/#{RUBY_VERSION} gccl/#{Google::Cloud::Bigquery::VERSION}"
|
68
|
+
service.request_options.quota_project = @quota_project if @quota_project
|
67
69
|
service.authorization = @credentials.client
|
68
70
|
service.root_url = host if host
|
69
71
|
service
|
@@ -138,6 +140,8 @@ module Google
|
|
138
140
|
end
|
139
141
|
end
|
140
142
|
|
143
|
+
##
|
144
|
+
# Gets the specified table resource by full table reference.
|
141
145
|
def get_project_table project_id, dataset_id, table_id
|
142
146
|
# The get operation is considered idempotent
|
143
147
|
execute backoff: true do
|
@@ -151,10 +155,7 @@ module Google
|
|
151
155
|
# it only returns the table resource,
|
152
156
|
# which describes the structure of this table.
|
153
157
|
def get_table dataset_id, table_id
|
154
|
-
|
155
|
-
execute backoff: true do
|
156
|
-
get_project_table @project, dataset_id, table_id
|
157
|
-
end
|
158
|
+
get_project_table @project, dataset_id, table_id
|
158
159
|
end
|
159
160
|
|
160
161
|
##
|
@@ -250,18 +251,21 @@ module Google
|
|
250
251
|
end
|
251
252
|
end
|
252
253
|
|
253
|
-
# Gets the specified model resource by model
|
254
|
-
|
255
|
-
# it only returns the model resource,
|
256
|
-
# which describes the structure of this model.
|
257
|
-
def get_model dataset_id, model_id
|
254
|
+
# Gets the specified model resource by full model reference.
|
255
|
+
def get_project_model project_id, dataset_id, model_id
|
258
256
|
# The get operation is considered idempotent
|
259
257
|
execute backoff: true do
|
260
|
-
json_txt = service.get_model
|
258
|
+
json_txt = service.get_model project_id, dataset_id, model_id, options: { skip_deserialization: true }
|
261
259
|
JSON.parse json_txt, symbolize_names: true
|
262
260
|
end
|
263
261
|
end
|
264
262
|
|
263
|
+
# Gets the specified model resource by model ID. This method does not return the data in the model, it only
|
264
|
+
# returns the model resource, which describes the structure of this model.
|
265
|
+
def get_model dataset_id, model_id
|
266
|
+
get_project_model @project, dataset_id, model_id
|
267
|
+
end
|
268
|
+
|
265
269
|
##
|
266
270
|
# Updates information in an existing model, replacing fields that
|
267
271
|
# are provided in the submitted model resource.
|
@@ -339,14 +343,16 @@ module Google
|
|
339
343
|
##
|
340
344
|
# Lists all jobs in the specified project to which you have
|
341
345
|
# been granted the READER job role.
|
342
|
-
def list_jobs all: nil,
|
346
|
+
def list_jobs all: nil, token: nil, max: nil, filter: nil, min_created_at: nil, max_created_at: nil,
|
347
|
+
parent_job_id: nil
|
343
348
|
# The list operation is considered idempotent
|
344
349
|
min_creation_time = Convert.time_to_millis min_created_at
|
345
350
|
max_creation_time = Convert.time_to_millis max_created_at
|
346
351
|
execute backoff: true do
|
347
352
|
service.list_jobs @project, all_users: all, max_results: max,
|
348
353
|
page_token: token, projection: "full", state_filter: filter,
|
349
|
-
min_creation_time: min_creation_time, max_creation_time: max_creation_time
|
354
|
+
min_creation_time: min_creation_time, max_creation_time: max_creation_time,
|
355
|
+
parent_job_id: parent_job_id
|
350
356
|
end
|
351
357
|
end
|
352
358
|
|
@@ -820,12 +820,19 @@ module Google
|
|
820
820
|
# @param [Hash<String, String>] labels A hash containing key/value
|
821
821
|
# pairs.
|
822
822
|
#
|
823
|
-
#
|
824
|
-
#
|
825
|
-
#
|
826
|
-
# *
|
827
|
-
# *
|
828
|
-
#
|
823
|
+
# The labels applied to a resource must meet the following requirements:
|
824
|
+
#
|
825
|
+
# * Each resource can have multiple labels, up to a maximum of 64.
|
826
|
+
# * Each label must be a key-value pair.
|
827
|
+
# * Keys have a minimum length of 1 character and a maximum length of
|
828
|
+
# 63 characters, and cannot be empty. Values can be empty, and have
|
829
|
+
# a maximum length of 63 characters.
|
830
|
+
# * Keys and values can contain only lowercase letters, numeric characters,
|
831
|
+
# underscores, and dashes. All characters must use UTF-8 encoding, and
|
832
|
+
# international characters are allowed.
|
833
|
+
# * The key portion of a label must be unique. However, you can use the
|
834
|
+
# same key with multiple resources.
|
835
|
+
# * Keys must start with a lowercase letter or international character.
|
829
836
|
#
|
830
837
|
# @example
|
831
838
|
# require "google/cloud/bigquery"
|
@@ -1181,12 +1188,20 @@ module Google
|
|
1181
1188
|
# SQL](https://cloud.google.com/bigquery/docs/reference/legacy-sql)
|
1182
1189
|
# dialect. Optional. The default value is false.
|
1183
1190
|
# @param [Array<String>, String] udfs User-defined function resources
|
1184
|
-
# used in
|
1185
|
-
# Google Cloud Storage URI (`gs://bucket/path`), or an inline resource
|
1191
|
+
# used in a legacy SQL query. May be either a code resource to load from
|
1192
|
+
# a Google Cloud Storage URI (`gs://bucket/path`), or an inline resource
|
1186
1193
|
# that contains code for a user-defined function (UDF). Providing an
|
1187
1194
|
# inline code resource is equivalent to providing a URI for a file
|
1188
|
-
# containing the same code.
|
1189
|
-
#
|
1195
|
+
# containing the same code.
|
1196
|
+
#
|
1197
|
+
# This parameter is used for defining User Defined Function (UDF)
|
1198
|
+
# resources only when using legacy SQL. Users of standard SQL should
|
1199
|
+
# leverage either DDL (e.g. `CREATE [TEMPORARY] FUNCTION ...`) or the
|
1200
|
+
# Routines API to define UDF resources.
|
1201
|
+
#
|
1202
|
+
# For additional information on migrating, see: [Migrating to
|
1203
|
+
# standard SQL - Differences in user-defined JavaScript
|
1204
|
+
# functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/migrating-from-legacy-sql#differences_in_user-defined_javascript_functions)
|
1190
1205
|
#
|
1191
1206
|
# @example
|
1192
1207
|
# require "google/cloud/bigquery"
|
@@ -1278,12 +1293,13 @@ module Google
|
|
1278
1293
|
# table = dataset.table "my_table"
|
1279
1294
|
#
|
1280
1295
|
# data = table.data
|
1296
|
+
#
|
1297
|
+
# # Iterate over the first page of results
|
1281
1298
|
# data.each do |row|
|
1282
|
-
# puts row[:
|
1283
|
-
# end
|
1284
|
-
# if data.next?
|
1285
|
-
# more_data = data.next if data.next?
|
1299
|
+
# puts row[:name]
|
1286
1300
|
# end
|
1301
|
+
# # Retrieve the next page of results
|
1302
|
+
# data = data.next if data.next?
|
1287
1303
|
#
|
1288
1304
|
# @example Retrieve all rows of data: (See {Data#all})
|
1289
1305
|
# require "google/cloud/bigquery"
|
@@ -1293,8 +1309,9 @@ module Google
|
|
1293
1309
|
# table = dataset.table "my_table"
|
1294
1310
|
#
|
1295
1311
|
# data = table.data
|
1312
|
+
#
|
1296
1313
|
# data.all do |row|
|
1297
|
-
# puts row[:
|
1314
|
+
# puts row[:name]
|
1298
1315
|
# end
|
1299
1316
|
#
|
1300
1317
|
# @!group Data
|
@@ -1360,13 +1377,21 @@ module Google
|
|
1360
1377
|
# is 1,024 characters. If `job_id` is provided, then `prefix` will not
|
1361
1378
|
# be used.
|
1362
1379
|
# @param [Hash] labels A hash of user-provided labels associated with
|
1363
|
-
# the job. You can use these to organize and group your jobs.
|
1364
|
-
#
|
1365
|
-
#
|
1366
|
-
#
|
1367
|
-
#
|
1368
|
-
#
|
1369
|
-
#
|
1380
|
+
# the job. You can use these to organize and group your jobs.
|
1381
|
+
#
|
1382
|
+
# The labels applied to a resource must meet the following requirements:
|
1383
|
+
#
|
1384
|
+
# * Each resource can have multiple labels, up to a maximum of 64.
|
1385
|
+
# * Each label must be a key-value pair.
|
1386
|
+
# * Keys have a minimum length of 1 character and a maximum length of
|
1387
|
+
# 63 characters, and cannot be empty. Values can be empty, and have
|
1388
|
+
# a maximum length of 63 characters.
|
1389
|
+
# * Keys and values can contain only lowercase letters, numeric characters,
|
1390
|
+
# underscores, and dashes. All characters must use UTF-8 encoding, and
|
1391
|
+
# international characters are allowed.
|
1392
|
+
# * The key portion of a label must be unique. However, you can use the
|
1393
|
+
# same key with multiple resources.
|
1394
|
+
# * Keys must start with a lowercase letter or international character.
|
1370
1395
|
# @param [Boolean] dryrun If set, don't actually run this job. Behavior
|
1371
1396
|
# is undefined however for non-query jobs and may result in an error.
|
1372
1397
|
# Deprecated.
|
@@ -1501,11 +1526,11 @@ module Google
|
|
1501
1526
|
# The geographic location for the job ("US", "EU", etc.) can be set via
|
1502
1527
|
# {ExtractJob::Updater#location=} in a block passed to this method. If
|
1503
1528
|
# the table is a full resource representation (see {#resource_full?}),
|
1504
|
-
# the location of the job will be
|
1529
|
+
# the location of the job will automatically be set to the location of
|
1505
1530
|
# the table.
|
1506
1531
|
#
|
1507
|
-
# @see https://cloud.google.com/bigquery/exporting-data
|
1508
|
-
# Exporting
|
1532
|
+
# @see https://cloud.google.com/bigquery/docs/exporting-data
|
1533
|
+
# Exporting table data
|
1509
1534
|
#
|
1510
1535
|
# @param [Google::Cloud::Storage::File, String, Array<String>]
|
1511
1536
|
# extract_url The Google Storage file or file URI pattern(s) to which
|
@@ -1541,13 +1566,21 @@ module Google
|
|
1541
1566
|
# is 1,024 characters. If `job_id` is provided, then `prefix` will not
|
1542
1567
|
# be used.
|
1543
1568
|
# @param [Hash] labels A hash of user-provided labels associated with
|
1544
|
-
# the job. You can use these to organize and group your jobs.
|
1545
|
-
#
|
1546
|
-
#
|
1547
|
-
#
|
1548
|
-
#
|
1549
|
-
#
|
1550
|
-
#
|
1569
|
+
# the job. You can use these to organize and group your jobs.
|
1570
|
+
#
|
1571
|
+
# The labels applied to a resource must meet the following requirements:
|
1572
|
+
#
|
1573
|
+
# * Each resource can have multiple labels, up to a maximum of 64.
|
1574
|
+
# * Each label must be a key-value pair.
|
1575
|
+
# * Keys have a minimum length of 1 character and a maximum length of
|
1576
|
+
# 63 characters, and cannot be empty. Values can be empty, and have
|
1577
|
+
# a maximum length of 63 characters.
|
1578
|
+
# * Keys and values can contain only lowercase letters, numeric characters,
|
1579
|
+
# underscores, and dashes. All characters must use UTF-8 encoding, and
|
1580
|
+
# international characters are allowed.
|
1581
|
+
# * The key portion of a label must be unique. However, you can use the
|
1582
|
+
# same key with multiple resources.
|
1583
|
+
# * Keys must start with a lowercase letter or international character.
|
1551
1584
|
# @param [Boolean] dryrun If set, don't actually run this job. Behavior
|
1552
1585
|
# is undefined however for non-query jobs and may result in an error.
|
1553
1586
|
# Deprecated.
|
@@ -1599,8 +1632,8 @@ module Google
|
|
1599
1632
|
# the location of the job will be automatically set to the location of
|
1600
1633
|
# the table.
|
1601
1634
|
#
|
1602
|
-
# @see https://cloud.google.com/bigquery/exporting-data
|
1603
|
-
# Exporting
|
1635
|
+
# @see https://cloud.google.com/bigquery/docs/exporting-data
|
1636
|
+
# Exporting table data
|
1604
1637
|
#
|
1605
1638
|
# @param [Google::Cloud::Storage::File, String, Array<String>]
|
1606
1639
|
# extract_url The Google Storage file or file URI pattern(s) to which
|
@@ -1781,13 +1814,21 @@ module Google
|
|
1781
1814
|
# is 1,024 characters. If `job_id` is provided, then `prefix` will not
|
1782
1815
|
# be used.
|
1783
1816
|
# @param [Hash] labels A hash of user-provided labels associated with
|
1784
|
-
# the job. You can use these to organize and group your jobs.
|
1785
|
-
#
|
1786
|
-
#
|
1787
|
-
#
|
1788
|
-
#
|
1789
|
-
#
|
1790
|
-
#
|
1817
|
+
# the job. You can use these to organize and group your jobs.
|
1818
|
+
#
|
1819
|
+
# The labels applied to a resource must meet the following requirements:
|
1820
|
+
#
|
1821
|
+
# * Each resource can have multiple labels, up to a maximum of 64.
|
1822
|
+
# * Each label must be a key-value pair.
|
1823
|
+
# * Keys have a minimum length of 1 character and a maximum length of
|
1824
|
+
# 63 characters, and cannot be empty. Values can be empty, and have
|
1825
|
+
# a maximum length of 63 characters.
|
1826
|
+
# * Keys and values can contain only lowercase letters, numeric characters,
|
1827
|
+
# underscores, and dashes. All characters must use UTF-8 encoding, and
|
1828
|
+
# international characters are allowed.
|
1829
|
+
# * The key portion of a label must be unique. However, you can use the
|
1830
|
+
# same key with multiple resources.
|
1831
|
+
# * Keys must start with a lowercase letter or international character.
|
1791
1832
|
# @param [Boolean] dryrun If set, don't actually run this job. Behavior
|
1792
1833
|
# is undefined however for non-query jobs and may result in an error.
|
1793
1834
|
# Deprecated.
|
@@ -34,9 +34,12 @@ module Google
|
|
34
34
|
# "WHERE time_of_date = @time",
|
35
35
|
# params: { time: fourpm }
|
36
36
|
#
|
37
|
+
# # Iterate over the first page of results
|
37
38
|
# data.each do |row|
|
38
39
|
# puts row[:name]
|
39
40
|
# end
|
41
|
+
# # Retrieve the next page of results
|
42
|
+
# data = data.next if data.next?
|
40
43
|
#
|
41
44
|
# @example Create Time with fractional seconds:
|
42
45
|
# require "google/cloud/bigquery"
|
@@ -49,9 +52,12 @@ module Google
|
|
49
52
|
# "WHERE time_of_date >= @time",
|
50
53
|
# params: { time: precise_time }
|
51
54
|
#
|
55
|
+
# # Iterate over the first page of results
|
52
56
|
# data.each do |row|
|
53
57
|
# puts row[:name]
|
54
58
|
# end
|
59
|
+
# # Retrieve the next page of results
|
60
|
+
# data = data.next if data.next?
|
55
61
|
#
|
56
62
|
Time = Struct.new :value
|
57
63
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: google-cloud-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.23.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mike Moore
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2020-
|
12
|
+
date: 2020-09-17 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: concurrent-ruby
|
@@ -115,14 +115,14 @@ dependencies:
|
|
115
115
|
requirements:
|
116
116
|
- - "~>"
|
117
117
|
- !ruby/object:Gem::Version
|
118
|
-
version: '5.
|
118
|
+
version: '5.14'
|
119
119
|
type: :development
|
120
120
|
prerelease: false
|
121
121
|
version_requirements: !ruby/object:Gem::Requirement
|
122
122
|
requirements:
|
123
123
|
- - "~>"
|
124
124
|
- !ruby/object:Gem::Version
|
125
|
-
version: '5.
|
125
|
+
version: '5.14'
|
126
126
|
- !ruby/object:Gem::Dependency
|
127
127
|
name: minitest-autotest
|
128
128
|
requirement: !ruby/object:Gem::Requirement
|
@@ -290,7 +290,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
290
290
|
- !ruby/object:Gem::Version
|
291
291
|
version: '0'
|
292
292
|
requirements: []
|
293
|
-
rubygems_version: 3.
|
293
|
+
rubygems_version: 3.1.4
|
294
294
|
signing_key:
|
295
295
|
specification_version: 4
|
296
296
|
summary: API Client library for Google BigQuery
|