google-cloud-bigquery 0.29.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +2 -2
- data/lib/google-cloud-bigquery.rb +12 -10
- data/lib/google/cloud/bigquery.rb +57 -17
- data/lib/google/cloud/bigquery/credentials.rb +31 -5
- data/lib/google/cloud/bigquery/data.rb +12 -12
- data/lib/google/cloud/bigquery/dataset.rb +281 -38
- data/lib/google/cloud/bigquery/dataset/access.rb +6 -6
- data/lib/google/cloud/bigquery/job.rb +1 -1
- data/lib/google/cloud/bigquery/project.rb +22 -7
- data/lib/google/cloud/bigquery/query_job.rb +6 -4
- data/lib/google/cloud/bigquery/service.rb +7 -5
- data/lib/google/cloud/bigquery/table.rb +487 -62
- data/lib/google/cloud/bigquery/table/async_inserter.rb +180 -5
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +21 -8
- data/lib/google/cloud/bigquery/view.rb +0 -739
@@ -33,9 +33,13 @@ module Google
|
|
33
33
|
# bigquery = Google::Cloud::Bigquery.new
|
34
34
|
# dataset = bigquery.dataset "my_dataset"
|
35
35
|
# table = dataset.table "my_table"
|
36
|
-
# inserter = table.insert_async do |
|
37
|
-
#
|
38
|
-
#
|
36
|
+
# inserter = table.insert_async do |result|
|
37
|
+
# if result.error?
|
38
|
+
# log_error result.error
|
39
|
+
# else
|
40
|
+
# log_insert "inserted #{result.insert_count} rows " \
|
41
|
+
# "with #{result.error_count} errors"
|
42
|
+
# end
|
39
43
|
# end
|
40
44
|
#
|
41
45
|
# rows = [
|
@@ -230,9 +234,11 @@ module Google
|
|
230
234
|
response = @table.insert batch_rows,
|
231
235
|
skip_invalid: @skip_invalid,
|
232
236
|
ignore_unknown: @ignore_unknown
|
233
|
-
|
237
|
+
result = Result.new response
|
234
238
|
rescue => e
|
235
|
-
|
239
|
+
result = Result.new nil, e
|
240
|
+
ensure
|
241
|
+
@callback.call result if @callback
|
236
242
|
end
|
237
243
|
end.execute
|
238
244
|
|
@@ -273,6 +279,175 @@ module Google
|
|
273
279
|
Convert.to_json_rows(rows).to_json.bytes.size
|
274
280
|
end
|
275
281
|
end
|
282
|
+
|
283
|
+
##
|
284
|
+
# AsyncInserter::Result
|
285
|
+
#
|
286
|
+
# Represents the result from BigQuery, including any error
|
287
|
+
# encountered, when data is asynchronously inserted into a table for
|
288
|
+
# near-immediate querying. See {Dataset#insert_async} and
|
289
|
+
# {Table#insert_async}.
|
290
|
+
#
|
291
|
+
# @see https://cloud.google.com/bigquery/streaming-data-into-bigquery
|
292
|
+
# Streaming Data Into BigQuery
|
293
|
+
#
|
294
|
+
# @attr_reader [Google::Cloud::Bigquery::InsertResponse, nil]
|
295
|
+
# insert_response The response from the insert operation if no
|
296
|
+
# error was encountered, or `nil` if the insert operation
|
297
|
+
# encountered an error.
|
298
|
+
# @attr_reader [Error, nil] error The error from the insert operation
|
299
|
+
# if any error was encountered, otherwise `nil`.
|
300
|
+
#
|
301
|
+
# @example
|
302
|
+
# require "google/cloud/bigquery"
|
303
|
+
#
|
304
|
+
# bigquery = Google::Cloud::Bigquery.new
|
305
|
+
# dataset = bigquery.dataset "my_dataset"
|
306
|
+
# table = dataset.table "my_table"
|
307
|
+
# inserter = table.insert_async do |result|
|
308
|
+
# if result.error?
|
309
|
+
# log_error result.error
|
310
|
+
# else
|
311
|
+
# log_insert "inserted #{result.insert_count} rows " \
|
312
|
+
# "with #{result.error_count} errors"
|
313
|
+
# end
|
314
|
+
# end
|
315
|
+
#
|
316
|
+
# rows = [
|
317
|
+
# { "first_name" => "Alice", "age" => 21 },
|
318
|
+
# { "first_name" => "Bob", "age" => 22 }
|
319
|
+
# ]
|
320
|
+
# inserter.insert rows
|
321
|
+
#
|
322
|
+
# inserter.stop.wait!
|
323
|
+
#
|
324
|
+
class Result
|
325
|
+
# @private
|
326
|
+
def initialize insert_response, error = nil
|
327
|
+
@insert_response = insert_response
|
328
|
+
@error = error
|
329
|
+
end
|
330
|
+
|
331
|
+
attr_reader :insert_response, :error
|
332
|
+
|
333
|
+
##
|
334
|
+
# Checks if an error is present, meaning that the insert operation
|
335
|
+
# encountered an error. Use {#error} to access the error. For
|
336
|
+
# row-level errors, see {#success?} and {#insert_errors}.
|
337
|
+
#
|
338
|
+
# @return [Boolean] `true` when an error is present, `false`
|
339
|
+
# otherwise.
|
340
|
+
#
|
341
|
+
def error?
|
342
|
+
!error.nil?
|
343
|
+
end
|
344
|
+
|
345
|
+
##
|
346
|
+
# Checks if the error count for row-level errors is zero, meaning
|
347
|
+
# that all of the rows were inserted. Use {#insert_errors} to access
|
348
|
+
# the row-level errors. To check for and access any operation-level
|
349
|
+
# error, use {#error?} and {#error}.
|
350
|
+
#
|
351
|
+
# @return [Boolean, nil] `true` when the error count is zero,
|
352
|
+
# `false` when the error count is positive, or `nil` if the insert
|
353
|
+
# operation encountered an error.
|
354
|
+
#
|
355
|
+
def success?
|
356
|
+
return nil if error?
|
357
|
+
insert_response.success?
|
358
|
+
end
|
359
|
+
|
360
|
+
|
361
|
+
##
|
362
|
+
# The count of rows in the response, minus the count of errors for
|
363
|
+
# rows that were not inserted.
|
364
|
+
#
|
365
|
+
# @return [Integer, nil] The number of rows inserted, or `nil` if
|
366
|
+
# the insert operation encountered an error.
|
367
|
+
#
|
368
|
+
def insert_count
|
369
|
+
return nil if error?
|
370
|
+
insert_response.insert_count
|
371
|
+
end
|
372
|
+
|
373
|
+
|
374
|
+
##
|
375
|
+
# The count of errors for rows that were not inserted.
|
376
|
+
#
|
377
|
+
# @return [Integer, nil] The number of errors, or `nil` if the
|
378
|
+
# insert operation encountered an error.
|
379
|
+
#
|
380
|
+
def error_count
|
381
|
+
return nil if error?
|
382
|
+
insert_response.error_count
|
383
|
+
end
|
384
|
+
|
385
|
+
##
|
386
|
+
# The error objects for rows that were not inserted.
|
387
|
+
#
|
388
|
+
# @return [Array<InsertError>, nil] An array containing error
|
389
|
+
# objects, or `nil` if the insert operation encountered an error.
|
390
|
+
#
|
391
|
+
def insert_errors
|
392
|
+
return nil if error?
|
393
|
+
insert_response.insert_errors
|
394
|
+
end
|
395
|
+
|
396
|
+
##
|
397
|
+
# The rows that were not inserted.
|
398
|
+
#
|
399
|
+
# @return [Array<Hash>, nil] An array of hash objects containing the
|
400
|
+
# row data, or `nil` if the insert operation encountered an error.
|
401
|
+
#
|
402
|
+
def error_rows
|
403
|
+
return nil if error?
|
404
|
+
insert_response.error_rows
|
405
|
+
end
|
406
|
+
|
407
|
+
##
|
408
|
+
# Returns the error object for a row that was not inserted.
|
409
|
+
#
|
410
|
+
# @param [Hash] row A hash containing the data for a row.
|
411
|
+
#
|
412
|
+
# @return [InsertError, nil] An error object, `nil` if no error is
|
413
|
+
# found in the response for the row, or `nil` if the insert
|
414
|
+
# operation encountered an error.
|
415
|
+
#
|
416
|
+
def insert_error_for row
|
417
|
+
return nil if error?
|
418
|
+
insert_response.insert_error_for row
|
419
|
+
end
|
420
|
+
|
421
|
+
##
|
422
|
+
# Returns the error hashes for a row that was not inserted. Each
|
423
|
+
# error hash contains the following keys: `reason`, `location`,
|
424
|
+
# `debugInfo`, and `message`.
|
425
|
+
#
|
426
|
+
# @param [Hash, nil] row A hash containing the data for a row.
|
427
|
+
#
|
428
|
+
# @return [Array<Hash>, nil] An array of error hashes, `nil` if no
|
429
|
+
# errors are found in the response for the row, or `nil` if the
|
430
|
+
# insert operation encountered an error.
|
431
|
+
#
|
432
|
+
def errors_for row
|
433
|
+
return nil if error?
|
434
|
+
insert_response.errors_for row
|
435
|
+
end
|
436
|
+
|
437
|
+
##
|
438
|
+
# Returns the index for a row that was not inserted.
|
439
|
+
#
|
440
|
+
# @param [Hash, nil] row A hash containing the data for a row.
|
441
|
+
#
|
442
|
+
# @return [Integer, nil] An error object, `nil` if no error is
|
443
|
+
# found in the response for the row, or `nil` if the insert
|
444
|
+
# operation encountered an error.
|
445
|
+
#
|
446
|
+
def index_for row
|
447
|
+
return nil if error?
|
448
|
+
insert_response.index_for row
|
449
|
+
end
|
450
|
+
end
|
276
451
|
end
|
277
452
|
end
|
278
453
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: google-cloud-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.30.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mike Moore
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-11-14 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: google-cloud-core
|
@@ -17,28 +17,42 @@ dependencies:
|
|
17
17
|
requirements:
|
18
18
|
- - "~>"
|
19
19
|
- !ruby/object:Gem::Version
|
20
|
-
version: '1.
|
20
|
+
version: '1.1'
|
21
21
|
type: :runtime
|
22
22
|
prerelease: false
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
24
24
|
requirements:
|
25
25
|
- - "~>"
|
26
26
|
- !ruby/object:Gem::Version
|
27
|
-
version: '1.
|
27
|
+
version: '1.1'
|
28
28
|
- !ruby/object:Gem::Dependency
|
29
29
|
name: google-api-client
|
30
30
|
requirement: !ruby/object:Gem::Requirement
|
31
31
|
requirements:
|
32
32
|
- - "~>"
|
33
33
|
- !ruby/object:Gem::Version
|
34
|
-
version: 0.
|
34
|
+
version: 0.17.0
|
35
|
+
type: :runtime
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "~>"
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: 0.17.0
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: googleauth
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - "~>"
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: 0.6.2
|
35
49
|
type: :runtime
|
36
50
|
prerelease: false
|
37
51
|
version_requirements: !ruby/object:Gem::Requirement
|
38
52
|
requirements:
|
39
53
|
- - "~>"
|
40
54
|
- !ruby/object:Gem::Version
|
41
|
-
version: 0.
|
55
|
+
version: 0.6.2
|
42
56
|
- !ruby/object:Gem::Dependency
|
43
57
|
name: concurrent-ruby
|
44
58
|
requirement: !ruby/object:Gem::Requirement
|
@@ -216,7 +230,6 @@ files:
|
|
216
230
|
- lib/google/cloud/bigquery/table/list.rb
|
217
231
|
- lib/google/cloud/bigquery/time.rb
|
218
232
|
- lib/google/cloud/bigquery/version.rb
|
219
|
-
- lib/google/cloud/bigquery/view.rb
|
220
233
|
homepage: https://github.com/GoogleCloudPlatform/google-cloud-ruby/tree/master/google-cloud-bigquery
|
221
234
|
licenses:
|
222
235
|
- Apache-2.0
|
@@ -237,7 +250,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
237
250
|
version: '0'
|
238
251
|
requirements: []
|
239
252
|
rubyforge_project:
|
240
|
-
rubygems_version: 2.
|
253
|
+
rubygems_version: 2.7.2
|
241
254
|
signing_key:
|
242
255
|
specification_version: 4
|
243
256
|
summary: API Client library for Google BigQuery
|
@@ -1,739 +0,0 @@
|
|
1
|
-
# Copyright 2015 Google Inc. All rights reserved.
|
2
|
-
#
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
-
# you may not use this file except in compliance with the License.
|
5
|
-
# You may obtain a copy of the License at
|
6
|
-
#
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
#
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
# See the License for the specific language governing permissions and
|
13
|
-
# limitations under the License.
|
14
|
-
|
15
|
-
|
16
|
-
require "google/cloud/errors"
|
17
|
-
require "google/cloud/bigquery/service"
|
18
|
-
require "google/cloud/bigquery/table/list"
|
19
|
-
require "google/apis/bigquery_v2"
|
20
|
-
|
21
|
-
module Google
|
22
|
-
module Cloud
|
23
|
-
module Bigquery
|
24
|
-
##
|
25
|
-
# # View
|
26
|
-
#
|
27
|
-
# A view is a virtual table defined by a SQL query. You can query views in
|
28
|
-
# the browser tool, or by using a query job.
|
29
|
-
#
|
30
|
-
# BigQuery's views are logical views, not materialized views, which means
|
31
|
-
# that the query that defines the view is re-executed every time the view
|
32
|
-
# is queried. Queries are billed according to the total amount of data in
|
33
|
-
# all table fields referenced directly or indirectly by the top-level
|
34
|
-
# query.
|
35
|
-
#
|
36
|
-
# @example
|
37
|
-
# require "google/cloud/bigquery"
|
38
|
-
#
|
39
|
-
# bigquery = Google::Cloud::Bigquery.new
|
40
|
-
# dataset = bigquery.dataset "my_dataset"
|
41
|
-
# view = dataset.create_view "my_view",
|
42
|
-
# "SELECT name, age FROM `my_project.my_dataset.my_table`"
|
43
|
-
#
|
44
|
-
class View
|
45
|
-
##
|
46
|
-
# @private The Service object.
|
47
|
-
attr_accessor :service
|
48
|
-
|
49
|
-
##
|
50
|
-
# @private The Google API Client object.
|
51
|
-
attr_accessor :gapi
|
52
|
-
|
53
|
-
##
|
54
|
-
# @private Create an empty View object.
|
55
|
-
def initialize
|
56
|
-
@service = nil
|
57
|
-
@gapi = Google::Apis::BigqueryV2::Table.new
|
58
|
-
end
|
59
|
-
|
60
|
-
##
|
61
|
-
# A unique ID for this view.
|
62
|
-
#
|
63
|
-
# @return [String] The ID must contain only letters (a-z, A-Z), numbers
|
64
|
-
# (0-9), or underscores (_). The maximum length is 1,024 characters.
|
65
|
-
#
|
66
|
-
# @!group Attributes
|
67
|
-
#
|
68
|
-
def table_id
|
69
|
-
@gapi.table_reference.table_id
|
70
|
-
end
|
71
|
-
|
72
|
-
##
|
73
|
-
# The ID of the `Dataset` containing this view.
|
74
|
-
#
|
75
|
-
# @return [String] The ID must contain only letters (a-z, A-Z), numbers
|
76
|
-
# (0-9), or underscores (_). The maximum length is 1,024 characters.
|
77
|
-
#
|
78
|
-
# @!group Attributes
|
79
|
-
#
|
80
|
-
def dataset_id
|
81
|
-
@gapi.table_reference.dataset_id
|
82
|
-
end
|
83
|
-
|
84
|
-
##
|
85
|
-
# The ID of the `Project` containing this view.
|
86
|
-
#
|
87
|
-
# @return [String] The project ID.
|
88
|
-
#
|
89
|
-
# @!group Attributes
|
90
|
-
#
|
91
|
-
def project_id
|
92
|
-
@gapi.table_reference.project_id
|
93
|
-
end
|
94
|
-
|
95
|
-
##
|
96
|
-
# @private The gapi fragment containing the Project ID, Dataset ID, and
|
97
|
-
# Table ID as a camel-cased hash.
|
98
|
-
def table_ref
|
99
|
-
table_ref = @gapi.table_reference
|
100
|
-
table_ref = table_ref.to_hash if table_ref.respond_to? :to_hash
|
101
|
-
table_ref
|
102
|
-
end
|
103
|
-
|
104
|
-
##
|
105
|
-
# The combined Project ID, Dataset ID, and Table ID for this view, in
|
106
|
-
# the format specified by the [Legacy SQL Query
|
107
|
-
# Reference](https://cloud.google.com/bigquery/query-reference#from):
|
108
|
-
# `project_name:datasetId.tableId`. To use this value in queries see
|
109
|
-
# {#query_id}.
|
110
|
-
#
|
111
|
-
# @!group Attributes
|
112
|
-
#
|
113
|
-
def id
|
114
|
-
@gapi.id
|
115
|
-
end
|
116
|
-
|
117
|
-
##
|
118
|
-
# The value returned by {#id}, wrapped in square brackets if the Project
|
119
|
-
# ID contains dashes, as specified by the [Query
|
120
|
-
# Reference](https://cloud.google.com/bigquery/query-reference#from).
|
121
|
-
# Useful in queries.
|
122
|
-
#
|
123
|
-
# @param [Boolean] standard_sql Specifies whether to use BigQuery's
|
124
|
-
# [standard
|
125
|
-
# SQL](https://cloud.google.com/bigquery/docs/reference/standard-sql/)
|
126
|
-
# dialect. Optional. The default value is true.
|
127
|
-
# @param [Boolean] legacy_sql Specifies whether to use BigQuery's
|
128
|
-
# [legacy
|
129
|
-
# SQL](https://cloud.google.com/bigquery/docs/reference/legacy-sql)
|
130
|
-
# dialect. Optional. The default value is false.
|
131
|
-
#
|
132
|
-
# @example
|
133
|
-
# require "google/cloud/bigquery"
|
134
|
-
#
|
135
|
-
# bigquery = Google::Cloud::Bigquery.new
|
136
|
-
# dataset = bigquery.dataset "my_dataset"
|
137
|
-
# view = dataset.table "my_view"
|
138
|
-
#
|
139
|
-
# data = bigquery.query "SELECT name FROM #{view.query_id}"
|
140
|
-
#
|
141
|
-
# @!group Attributes
|
142
|
-
#
|
143
|
-
def query_id standard_sql: nil, legacy_sql: nil
|
144
|
-
if Convert.resolve_legacy_sql standard_sql, legacy_sql
|
145
|
-
"[#{id}]"
|
146
|
-
else
|
147
|
-
"`#{project_id}.#{dataset_id}.#{table_id}`"
|
148
|
-
end
|
149
|
-
end
|
150
|
-
|
151
|
-
##
|
152
|
-
# The name of the view.
|
153
|
-
#
|
154
|
-
# @return [String] The friendly name.
|
155
|
-
#
|
156
|
-
# @!group Attributes
|
157
|
-
#
|
158
|
-
def name
|
159
|
-
@gapi.friendly_name
|
160
|
-
end
|
161
|
-
|
162
|
-
##
|
163
|
-
# Updates the name of the view.
|
164
|
-
#
|
165
|
-
# @param [String] new_name The new friendly name.
|
166
|
-
#
|
167
|
-
# @!group Attributes
|
168
|
-
#
|
169
|
-
def name= new_name
|
170
|
-
@gapi.update! friendly_name: new_name
|
171
|
-
patch_gapi! :friendly_name
|
172
|
-
end
|
173
|
-
|
174
|
-
##
|
175
|
-
# The ETag hash of the view.
|
176
|
-
#
|
177
|
-
# @return [String] The ETag hash.
|
178
|
-
#
|
179
|
-
# @!group Attributes
|
180
|
-
#
|
181
|
-
def etag
|
182
|
-
ensure_full_data!
|
183
|
-
@gapi.etag
|
184
|
-
end
|
185
|
-
|
186
|
-
##
|
187
|
-
# A URL that can be used to access the view using the REST API.
|
188
|
-
#
|
189
|
-
# @return [String] A REST URL for the resource.
|
190
|
-
#
|
191
|
-
# @!group Attributes
|
192
|
-
#
|
193
|
-
def api_url
|
194
|
-
ensure_full_data!
|
195
|
-
@gapi.self_link
|
196
|
-
end
|
197
|
-
|
198
|
-
##
|
199
|
-
# A user-friendly description of the view.
|
200
|
-
#
|
201
|
-
# @return [String] The description.
|
202
|
-
#
|
203
|
-
# @!group Attributes
|
204
|
-
#
|
205
|
-
def description
|
206
|
-
ensure_full_data!
|
207
|
-
@gapi.description
|
208
|
-
end
|
209
|
-
|
210
|
-
##
|
211
|
-
# Updates the user-friendly description of the view.
|
212
|
-
#
|
213
|
-
# @param [String] new_description The new user-friendly description.
|
214
|
-
#
|
215
|
-
# @!group Attributes
|
216
|
-
#
|
217
|
-
def description= new_description
|
218
|
-
@gapi.update! description: new_description
|
219
|
-
patch_gapi! :description
|
220
|
-
end
|
221
|
-
|
222
|
-
##
|
223
|
-
# The time when this view was created.
|
224
|
-
#
|
225
|
-
# @return [Time, nil] The creation time.
|
226
|
-
#
|
227
|
-
# @!group Attributes
|
228
|
-
#
|
229
|
-
def created_at
|
230
|
-
ensure_full_data!
|
231
|
-
begin
|
232
|
-
::Time.at(Integer(@gapi.creation_time) / 1000.0)
|
233
|
-
rescue
|
234
|
-
nil
|
235
|
-
end
|
236
|
-
end
|
237
|
-
|
238
|
-
##
|
239
|
-
# The time when this view expires.
|
240
|
-
# If not present, the view will persist indefinitely.
|
241
|
-
# Expired views will be deleted and their storage reclaimed.
|
242
|
-
#
|
243
|
-
# @return [Time, nil] The expiration time.
|
244
|
-
#
|
245
|
-
# @!group Attributes
|
246
|
-
#
|
247
|
-
def expires_at
|
248
|
-
ensure_full_data!
|
249
|
-
begin
|
250
|
-
::Time.at(Integer(@gapi.expiration_time) / 1000.0)
|
251
|
-
rescue
|
252
|
-
nil
|
253
|
-
end
|
254
|
-
end
|
255
|
-
|
256
|
-
##
|
257
|
-
# The date when this view was last modified.
|
258
|
-
#
|
259
|
-
# @return [Time, nil] The last modified time.
|
260
|
-
#
|
261
|
-
# @!group Attributes
|
262
|
-
#
|
263
|
-
def modified_at
|
264
|
-
ensure_full_data!
|
265
|
-
begin
|
266
|
-
::Time.at(Integer(@gapi.last_modified_time) / 1000.0)
|
267
|
-
rescue
|
268
|
-
nil
|
269
|
-
end
|
270
|
-
end
|
271
|
-
|
272
|
-
##
|
273
|
-
# Checks if the view's type is "TABLE".
|
274
|
-
#
|
275
|
-
# @return [Boolean] `true` when the type is `TABLE`, `false` otherwise.
|
276
|
-
#
|
277
|
-
# @!group Attributes
|
278
|
-
#
|
279
|
-
def table?
|
280
|
-
@gapi.type == "TABLE"
|
281
|
-
end
|
282
|
-
|
283
|
-
##
|
284
|
-
# Checks if the view's type is "VIEW".
|
285
|
-
#
|
286
|
-
# @return [Boolean] `true` when the type is `VIEW`, `false` otherwise.
|
287
|
-
#
|
288
|
-
# @!group Attributes
|
289
|
-
#
|
290
|
-
def view?
|
291
|
-
@gapi.type == "VIEW"
|
292
|
-
end
|
293
|
-
|
294
|
-
##
|
295
|
-
# Checks if the view's type is "EXTERNAL".
|
296
|
-
#
|
297
|
-
# @return [Boolean] `true` when the type is `EXTERNAL`, `false`
|
298
|
-
# otherwise.
|
299
|
-
#
|
300
|
-
# @!group Attributes
|
301
|
-
#
|
302
|
-
def external?
|
303
|
-
@gapi.type == "EXTERNAL"
|
304
|
-
end
|
305
|
-
|
306
|
-
##
|
307
|
-
# The geographic location where the view should reside. Possible
|
308
|
-
# values include `EU` and `US`. The default value is `US`.
|
309
|
-
#
|
310
|
-
# @return [String] The location code.
|
311
|
-
#
|
312
|
-
# @!group Attributes
|
313
|
-
#
|
314
|
-
def location
|
315
|
-
ensure_full_data!
|
316
|
-
@gapi.location
|
317
|
-
end
|
318
|
-
|
319
|
-
##
|
320
|
-
# A hash of user-provided labels associated with this view. Labels
|
321
|
-
# are used to organize and group views and views. See [Using
|
322
|
-
# Labels](https://cloud.google.com/bigquery/docs/labels).
|
323
|
-
#
|
324
|
-
# The returned hash is frozen and changes are not allowed. Use
|
325
|
-
# {#labels=} to replace the entire hash.
|
326
|
-
#
|
327
|
-
# @return [Hash<String, String>] A hash containing key/value pairs.
|
328
|
-
#
|
329
|
-
# @example
|
330
|
-
# require "google/cloud/bigquery"
|
331
|
-
#
|
332
|
-
# bigquery = Google::Cloud::Bigquery.new
|
333
|
-
# dataset = bigquery.dataset "my_dataset"
|
334
|
-
# view = dataset.table "my_view"
|
335
|
-
#
|
336
|
-
# labels = view.labels
|
337
|
-
# labels["department"] #=> "shipping"
|
338
|
-
#
|
339
|
-
# @!group Attributes
|
340
|
-
#
|
341
|
-
def labels
|
342
|
-
m = @gapi.labels
|
343
|
-
m = m.to_h if m.respond_to? :to_h
|
344
|
-
m.dup.freeze
|
345
|
-
end
|
346
|
-
|
347
|
-
##
|
348
|
-
# Updates the hash of user-provided labels associated with this view.
|
349
|
-
# Labels are used to organize and group tables and views. See [Using
|
350
|
-
# Labels](https://cloud.google.com/bigquery/docs/labels).
|
351
|
-
#
|
352
|
-
# @param [Hash<String, String>] labels A hash containing key/value
|
353
|
-
# pairs.
|
354
|
-
#
|
355
|
-
# * Label keys and values can be no longer than 63 characters.
|
356
|
-
# * Label keys and values can contain only lowercase letters, numbers,
|
357
|
-
# underscores, hyphens, and international characters.
|
358
|
-
# * Label keys and values cannot exceed 128 bytes in size.
|
359
|
-
# * Label keys must begin with a letter.
|
360
|
-
# * Label keys must be unique within a view.
|
361
|
-
#
|
362
|
-
# @example
|
363
|
-
# require "google/cloud/bigquery"
|
364
|
-
#
|
365
|
-
# bigquery = Google::Cloud::Bigquery.new
|
366
|
-
# dataset = bigquery.dataset "my_dataset"
|
367
|
-
# view = dataset.table "my_view"
|
368
|
-
#
|
369
|
-
# view.labels = { "department" => "shipping" }
|
370
|
-
#
|
371
|
-
# @!group Attributes
|
372
|
-
#
|
373
|
-
def labels= labels
|
374
|
-
@gapi.labels = labels
|
375
|
-
patch_gapi! :labels
|
376
|
-
end
|
377
|
-
|
378
|
-
##
|
379
|
-
# The schema of the view.
|
380
|
-
#
|
381
|
-
# The returned object is frozen and changes are not allowed.
|
382
|
-
#
|
383
|
-
# @return [Schema] A schema object.
|
384
|
-
#
|
385
|
-
# @example
|
386
|
-
# require "google/cloud/bigquery"
|
387
|
-
#
|
388
|
-
# bigquery = Google::Cloud::Bigquery.new
|
389
|
-
# dataset = bigquery.dataset "my_dataset"
|
390
|
-
# view = dataset.table "my_view"
|
391
|
-
#
|
392
|
-
# schema = view.schema
|
393
|
-
# field = schema.field "name"
|
394
|
-
# field.required? #=> true
|
395
|
-
#
|
396
|
-
# @!group Attributes
|
397
|
-
#
|
398
|
-
def schema
|
399
|
-
ensure_full_data!
|
400
|
-
Schema.from_gapi(@gapi.schema).freeze
|
401
|
-
end
|
402
|
-
|
403
|
-
##
|
404
|
-
# The fields of the view, obtained from its schema.
|
405
|
-
#
|
406
|
-
# @return [Array<Schema::Field>] An array of field objects.
|
407
|
-
#
|
408
|
-
# @example
|
409
|
-
# require "google/cloud/bigquery"
|
410
|
-
#
|
411
|
-
# bigquery = Google::Cloud::Bigquery.new
|
412
|
-
# dataset = bigquery.dataset "my_dataset"
|
413
|
-
# view = dataset.table "my_view"
|
414
|
-
#
|
415
|
-
# view.fields.each do |field|
|
416
|
-
# puts field.name
|
417
|
-
# end
|
418
|
-
#
|
419
|
-
# @!group Attributes
|
420
|
-
#
|
421
|
-
def fields
|
422
|
-
schema.fields
|
423
|
-
end
|
424
|
-
|
425
|
-
##
|
426
|
-
# The names of the columns in the view, obtained from its schema.
|
427
|
-
#
|
428
|
-
# @return [Array<Symbol>] An array of column names.
|
429
|
-
#
|
430
|
-
# @example
|
431
|
-
# require "google/cloud/bigquery"
|
432
|
-
#
|
433
|
-
# bigquery = Google::Cloud::Bigquery.new
|
434
|
-
# dataset = bigquery.dataset "my_dataset"
|
435
|
-
# view = dataset.table "my_view"
|
436
|
-
#
|
437
|
-
# view.headers.each do |header|
|
438
|
-
# puts header
|
439
|
-
# end
|
440
|
-
#
|
441
|
-
# @!group Attributes
|
442
|
-
#
|
443
|
-
def headers
|
444
|
-
schema.headers
|
445
|
-
end
|
446
|
-
|
447
|
-
##
|
448
|
-
# The query that executes each time the view is loaded.
|
449
|
-
#
|
450
|
-
# @return [String] The query that defines the view.
|
451
|
-
#
|
452
|
-
# @!group Attributes
|
453
|
-
#
|
454
|
-
def query
|
455
|
-
@gapi.view.query if @gapi.view
|
456
|
-
end
|
457
|
-
|
458
|
-
##
|
459
|
-
# Updates the query that executes each time the view is loaded.
|
460
|
-
#
|
461
|
-
# This sets the query using standard SQL. To specify legacy SQL or to
|
462
|
-
# use user-defined function resources use (#set_query) instead.
|
463
|
-
#
|
464
|
-
# @see https://cloud.google.com/bigquery/query-reference BigQuery Query
|
465
|
-
# Reference
|
466
|
-
#
|
467
|
-
# @param [String] new_query The query that defines the view.
|
468
|
-
#
|
469
|
-
# @example
|
470
|
-
# require "google/cloud/bigquery"
|
471
|
-
#
|
472
|
-
# bigquery = Google::Cloud::Bigquery.new
|
473
|
-
# dataset = bigquery.dataset "my_dataset"
|
474
|
-
# view = dataset.table "my_view"
|
475
|
-
#
|
476
|
-
# view.query = "SELECT first_name FROM " \
|
477
|
-
# "`my_project.my_dataset.my_table`"
|
478
|
-
#
|
479
|
-
# @!group Lifecycle
|
480
|
-
#
|
481
|
-
def query= new_query
|
482
|
-
set_query new_query
|
483
|
-
end
|
484
|
-
|
485
|
-
##
|
486
|
-
# Updates the query that executes each time the view is loaded. Allows
|
487
|
-
# setting of standard vs. legacy SQL and user-defined function
|
488
|
-
# resources.
|
489
|
-
#
|
490
|
-
# @see https://cloud.google.com/bigquery/query-reference BigQuery Query
|
491
|
-
# Reference
|
492
|
-
#
|
493
|
-
# @param [String] query The query that defines the view.
|
494
|
-
# @param [Boolean] standard_sql Specifies whether to use BigQuery's
|
495
|
-
# [standard
|
496
|
-
# SQL](https://cloud.google.com/bigquery/docs/reference/standard-sql/)
|
497
|
-
# dialect. Optional. The default value is true.
|
498
|
-
# @param [Boolean] legacy_sql Specifies whether to use BigQuery's
|
499
|
-
# [legacy
|
500
|
-
# SQL](https://cloud.google.com/bigquery/docs/reference/legacy-sql)
|
501
|
-
# dialect. Optional. The default value is false.
|
502
|
-
# @param [Array<String>, String] udfs User-defined function resources
|
503
|
-
# used in the query. May be either a code resource to load from a
|
504
|
-
# Google Cloud Storage URI (`gs://bucket/path`), or an inline resource
|
505
|
-
# that contains code for a user-defined function (UDF). Providing an
|
506
|
-
# inline code resource is equivalent to providing a URI for a file
|
507
|
-
# containing the same code. See [User-Defined
|
508
|
-
# Functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/user-defined-functions).
|
509
|
-
#
|
510
|
-
# @example
|
511
|
-
# require "google/cloud/bigquery"
|
512
|
-
#
|
513
|
-
# bigquery = Google::Cloud::Bigquery.new
|
514
|
-
# dataset = bigquery.dataset "my_dataset"
|
515
|
-
# view = dataset.table "my_view"
|
516
|
-
#
|
517
|
-
# view.set_query "SELECT first_name FROM " \
|
518
|
-
# "`my_project.my_dataset.my_table`",
|
519
|
-
# standard_sql: true
|
520
|
-
#
|
521
|
-
# @!group Lifecycle
|
522
|
-
#
|
523
|
-
def set_query query, standard_sql: nil, legacy_sql: nil, udfs: nil
|
524
|
-
@gapi.view = Google::Apis::BigqueryV2::ViewDefinition.new \
|
525
|
-
query: query,
|
526
|
-
use_legacy_sql: Convert.resolve_legacy_sql(standard_sql,
|
527
|
-
legacy_sql),
|
528
|
-
user_defined_function_resources: udfs_gapi(udfs)
|
529
|
-
patch_view_gapi!
|
530
|
-
end
|
531
|
-
|
532
|
-
##
|
533
|
-
# Checks if the view's query is using legacy sql.
|
534
|
-
#
|
535
|
-
# @return [Boolean] `true` when legacy sql is used, `false` otherwise.
|
536
|
-
#
|
537
|
-
# @!group Attributes
|
538
|
-
#
|
539
|
-
def query_legacy_sql?
|
540
|
-
val = @gapi.view.use_legacy_sql
|
541
|
-
return true if val.nil?
|
542
|
-
val
|
543
|
-
end
|
544
|
-
|
545
|
-
##
|
546
|
-
# Checks if the view's query is using standard sql.
|
547
|
-
#
|
548
|
-
# @return [Boolean] `true` when standard sql is used, `false` otherwise.
|
549
|
-
#
|
550
|
-
# @!group Attributes
|
551
|
-
#
|
552
|
-
def query_standard_sql?
|
553
|
-
!query_legacy_sql?
|
554
|
-
end
|
555
|
-
|
556
|
-
##
|
557
|
-
# The user-defined function resources used in the view's query. May be
|
558
|
-
# either a code resource to load from a Google Cloud Storage URI
|
559
|
-
# (`gs://bucket/path`), or an inline resource that contains code for a
|
560
|
-
# user-defined function (UDF). Providing an inline code resource is
|
561
|
-
# equivalent to providing a URI for a file containing the same code. See
|
562
|
-
# [User-Defined
|
563
|
-
# Functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/user-defined-functions).
|
564
|
-
#
|
565
|
-
# @return [Array<String>] An array containing Google Cloud Storage URIs
|
566
|
-
# and/or inline source code.
|
567
|
-
#
|
568
|
-
# @!group Attributes
|
569
|
-
#
|
570
|
-
def query_udfs
|
571
|
-
udfs_gapi = @gapi.view.user_defined_function_resources
|
572
|
-
return [] if udfs_gapi.nil?
|
573
|
-
Array(udfs_gapi).map { |udf| udf.inline_code || udf.resource_uri }
|
574
|
-
end
|
575
|
-
|
576
|
-
##
|
577
|
-
# Runs a query to retrieve all data from the view, in a synchronous
|
578
|
-
# method that blocks for a response. In this method, a {QueryJob} is
|
579
|
-
# created and its results are saved to a temporary table, then read from
|
580
|
-
# the table. Timeouts and transient errors are generally handled as
|
581
|
-
# needed to complete the query.
|
582
|
-
#
|
583
|
-
# @param [Integer] max The maximum number of rows of data to return per
|
584
|
-
# page of results. Setting this flag to a small value such as 1000 and
|
585
|
-
# then paging through results might improve reliability when the query
|
586
|
-
# result set is large. In addition to this limit, responses are also
|
587
|
-
# limited to 10 MB. By default, there is no maximum row count, and
|
588
|
-
# only the byte limit applies.
|
589
|
-
# @param [Boolean] cache Whether to look for the result in the query
|
590
|
-
# cache. The query cache is a best-effort cache that will be flushed
|
591
|
-
# whenever tables in the query are modified. The default value is
|
592
|
-
# true. For more information, see [query
|
593
|
-
# caching](https://developers.google.com/bigquery/querying-data).
|
594
|
-
#
|
595
|
-
# @return [Google::Cloud::Bigquery::Data]
|
596
|
-
#
|
597
|
-
# @example
|
598
|
-
# require "google/cloud/bigquery"
|
599
|
-
#
|
600
|
-
# bigquery = Google::Cloud::Bigquery.new
|
601
|
-
# dataset = bigquery.dataset "my_dataset"
|
602
|
-
# view = dataset.table "my_view"
|
603
|
-
#
|
604
|
-
# data = view.data
|
605
|
-
# data.each do |row|
|
606
|
-
# puts row[:first_name]
|
607
|
-
# end
|
608
|
-
# more_data = data.next if data.next?
|
609
|
-
#
|
610
|
-
# @!group Data
|
611
|
-
#
|
612
|
-
def data max: nil, cache: true
|
613
|
-
sql = "SELECT * FROM #{query_id}"
|
614
|
-
ensure_service!
|
615
|
-
|
616
|
-
gapi = service.query_job sql, cache: cache
|
617
|
-
job = Job.from_gapi gapi, service
|
618
|
-
job.wait_until_done!
|
619
|
-
|
620
|
-
if job.failed?
|
621
|
-
begin
|
622
|
-
# raise to activate ruby exception cause handling
|
623
|
-
fail job.gapi_error
|
624
|
-
rescue => e
|
625
|
-
# wrap Google::Apis::Error with Google::Cloud::Error
|
626
|
-
raise Google::Cloud::Error.from_error(e)
|
627
|
-
end
|
628
|
-
end
|
629
|
-
|
630
|
-
job.data max: max
|
631
|
-
end
|
632
|
-
|
633
|
-
##
|
634
|
-
# Permanently deletes the view.
|
635
|
-
#
|
636
|
-
# @return [Boolean] Returns `true` if the view was deleted.
|
637
|
-
#
|
638
|
-
# @example
|
639
|
-
# require "google/cloud/bigquery"
|
640
|
-
#
|
641
|
-
# bigquery = Google::Cloud::Bigquery.new
|
642
|
-
# dataset = bigquery.dataset "my_dataset"
|
643
|
-
# view = dataset.table "my_view"
|
644
|
-
#
|
645
|
-
# view.delete
|
646
|
-
#
|
647
|
-
# @!group Lifecycle
|
648
|
-
#
|
649
|
-
def delete
|
650
|
-
ensure_service!
|
651
|
-
service.delete_table dataset_id, table_id
|
652
|
-
true
|
653
|
-
end
|
654
|
-
|
655
|
-
##
|
656
|
-
# Reloads the view with current data from the BigQuery service.
|
657
|
-
#
|
658
|
-
# @!group Lifecycle
|
659
|
-
#
|
660
|
-
def reload!
|
661
|
-
ensure_service!
|
662
|
-
gapi = service.get_table dataset_id, table_id
|
663
|
-
@gapi = gapi
|
664
|
-
end
|
665
|
-
alias_method :refresh!, :reload!
|
666
|
-
|
667
|
-
##
|
668
|
-
# @private New Table from a Google API Client object.
|
669
|
-
def self.from_gapi gapi, conn
|
670
|
-
new.tap do |f|
|
671
|
-
f.gapi = gapi
|
672
|
-
f.service = conn
|
673
|
-
end
|
674
|
-
end
|
675
|
-
|
676
|
-
protected
|
677
|
-
|
678
|
-
##
|
679
|
-
# Raise an error unless an active service is available.
|
680
|
-
def ensure_service!
|
681
|
-
fail "Must have active connection" unless service
|
682
|
-
end
|
683
|
-
|
684
|
-
def patch_gapi! *attributes
|
685
|
-
return if attributes.empty?
|
686
|
-
patch_args = Hash[attributes.map do |attr|
|
687
|
-
[attr, @gapi.send(attr)]
|
688
|
-
end]
|
689
|
-
patch_table_gapi patch_args
|
690
|
-
end
|
691
|
-
|
692
|
-
def patch_table_gapi patch_args
|
693
|
-
ensure_service!
|
694
|
-
patch_gapi = Google::Apis::BigqueryV2::Table.new patch_args
|
695
|
-
patch_gapi.etag = etag if etag
|
696
|
-
@gapi = service.patch_table dataset_id, table_id, patch_gapi
|
697
|
-
|
698
|
-
# TODO: restore original impl after acceptance test indicates that
|
699
|
-
# service etag bug is fixed
|
700
|
-
reload!
|
701
|
-
end
|
702
|
-
|
703
|
-
def patch_view_gapi!
|
704
|
-
patch_table_gapi view: @gapi.view
|
705
|
-
end
|
706
|
-
|
707
|
-
##
|
708
|
-
# Load the complete representation of the table if it has been
|
709
|
-
# only partially loaded by a request to the API list method.
|
710
|
-
def ensure_full_data!
|
711
|
-
reload_gapi! unless data_complete?
|
712
|
-
end
|
713
|
-
|
714
|
-
def reload_gapi!
|
715
|
-
ensure_service!
|
716
|
-
gapi = service.get_table dataset_id, table_id
|
717
|
-
@gapi = gapi
|
718
|
-
end
|
719
|
-
|
720
|
-
def data_complete?
|
721
|
-
@gapi.is_a? Google::Apis::BigqueryV2::Table
|
722
|
-
end
|
723
|
-
|
724
|
-
def udfs_gapi array_or_str
|
725
|
-
return [] if array_or_str.nil?
|
726
|
-
Array(array_or_str).map do |uri_or_code|
|
727
|
-
resource = Google::Apis::BigqueryV2::UserDefinedFunctionResource.new
|
728
|
-
if uri_or_code.start_with?("gs://")
|
729
|
-
resource.resource_uri = uri_or_code
|
730
|
-
else
|
731
|
-
resource.inline_code = uri_or_code
|
732
|
-
end
|
733
|
-
resource
|
734
|
-
end
|
735
|
-
end
|
736
|
-
end
|
737
|
-
end
|
738
|
-
end
|
739
|
-
end
|