google-cloud-bigquery 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,182 @@
1
+ # Copyright 2015 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ require "delegate"
17
+
18
+ module Google
19
+ module Cloud
20
+ module Bigquery
21
+ class Table
22
+ ##
23
+ # Table::List is a special case Array with additional values.
24
+ class List < DelegateClass(::Array)
25
+ ##
26
+ # If not empty, indicates that there are more records that match
27
+ # the request and this value should be passed to continue.
28
+ attr_accessor :token
29
+
30
+ # A hash of this page of results.
31
+ attr_accessor :etag
32
+
33
+ # Total number of tables in this collection.
34
+ attr_accessor :total
35
+
36
+ ##
37
+ # @private Create a new Table::List with an array of tables.
38
+ def initialize arr = []
39
+ super arr
40
+ end
41
+
42
+ ##
43
+ # Whether there is a next page of tables.
44
+ #
45
+ # @return [Boolean]
46
+ #
47
+ # @example
48
+ # require "google/cloud"
49
+ #
50
+ # gcloud = Google::Cloud.new
51
+ # bigquery = gcloud.bigquery
52
+ # dataset = bigquery.dataset "my_dataset"
53
+ #
54
+ # tables = dataset.tables
55
+ # if tables.next?
56
+ # next_tables = tables.next
57
+ # end
58
+ #
59
+ def next?
60
+ !token.nil?
61
+ end
62
+
63
+ ##
64
+ # Retrieve the next page of tables.
65
+ #
66
+ # @return [Table::List]
67
+ #
68
+ # @example
69
+ # require "google/cloud"
70
+ #
71
+ # gcloud = Google::Cloud.new
72
+ # bigquery = gcloud.bigquery
73
+ # dataset = bigquery.dataset "my_dataset"
74
+ #
75
+ # tables = dataset.tables
76
+ # if tables.next?
77
+ # next_tables = tables.next
78
+ # end
79
+ #
80
+ def next
81
+ return nil unless next?
82
+ ensure_service!
83
+ options = { token: token, max: @max }
84
+ gapi = @service.list_tables @dataset_id, options
85
+ self.class.from_gapi gapi, @service, @dataset_id, @max
86
+ end
87
+
88
+ ##
89
+ # Retrieves all tables by repeatedly loading {#next} until {#next?}
90
+ # returns `false`. Calls the given block once for each table, which is
91
+ # passed as the parameter.
92
+ #
93
+ # An Enumerator is returned if no block is given.
94
+ #
95
+ # This method may make several API calls until all tables are
96
+ # retrieved. Be sure to use as narrow a search criteria as possible.
97
+ # Please use with caution.
98
+ #
99
+ # @param [Integer] request_limit The upper limit of API requests to
100
+ # make to load all tables. Default is no limit.
101
+ # @yield [table] The block for accessing each table.
102
+ # @yieldparam [Table] table The table object.
103
+ #
104
+ # @return [Enumerator]
105
+ #
106
+ # @example Iterating each result by passing a block:
107
+ # require "google/cloud"
108
+ #
109
+ # gcloud = Google::Cloud.new
110
+ # bigquery = gcloud.bigquery
111
+ # dataset = bigquery.dataset "my_dataset"
112
+ #
113
+ # dataset.tables.all do |table|
114
+ # puts table.name
115
+ # end
116
+ #
117
+ # @example Using the enumerator by not passing a block:
118
+ # require "google/cloud"
119
+ #
120
+ # gcloud = Google::Cloud.new
121
+ # bigquery = gcloud.bigquery
122
+ # dataset = bigquery.dataset "my_dataset"
123
+ #
124
+ # all_names = dataset.tables.all.map do |table|
125
+ # table.name
126
+ # end
127
+ #
128
+ # @example Limit the number of API requests made:
129
+ # require "google/cloud"
130
+ #
131
+ # gcloud = Google::Cloud.new
132
+ # bigquery = gcloud.bigquery
133
+ # dataset = bigquery.dataset "my_dataset"
134
+ #
135
+ # dataset.tables.all(request_limit: 10) do |table|
136
+ # puts table.name
137
+ # end
138
+ #
139
+ def all request_limit: nil
140
+ request_limit = request_limit.to_i if request_limit
141
+ unless block_given?
142
+ return enum_for(:all, request_limit: request_limit)
143
+ end
144
+ results = self
145
+ loop do
146
+ results.each { |r| yield r }
147
+ if request_limit
148
+ request_limit -= 1
149
+ break if request_limit < 0
150
+ end
151
+ break unless results.next?
152
+ results = results.next
153
+ end
154
+ end
155
+
156
+ ##
157
+ # @private New Table::List from a response object.
158
+ def self.from_gapi gapi_list, service, dataset_id = nil, max = nil
159
+ tables = List.new(Array(gapi_list.tables).map do |gapi_object|
160
+ Table.from_gapi gapi_object, service
161
+ end)
162
+ tables.instance_variable_set :@token, gapi_list.next_page_token
163
+ tables.instance_variable_set :@etag, gapi_list.etag
164
+ tables.instance_variable_set :@total, gapi_list.total_items
165
+ tables.instance_variable_set :@service, service
166
+ tables.instance_variable_set :@dataset_id, dataset_id
167
+ tables.instance_variable_set :@max, max
168
+ tables
169
+ end
170
+
171
+ protected
172
+
173
+ ##
174
+ # Raise an error unless an active service is available.
175
+ def ensure_service!
176
+ fail "Must have active connection" unless @service
177
+ end
178
+ end
179
+ end
180
+ end
181
+ end
182
+ end
@@ -0,0 +1,22 @@
1
+ # Copyright 2016 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ module Google
17
+ module Cloud
18
+ module Bigquery
19
+ VERSION = "0.20.0"
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,478 @@
1
+ # Copyright 2015 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ require "google/cloud/errors"
17
+ require "google/cloud/bigquery/service"
18
+ require "google/cloud/bigquery/data"
19
+ require "google/cloud/bigquery/table/list"
20
+ require "google/apis/bigquery_v2"
21
+
22
+ module Google
23
+ module Cloud
24
+ module Bigquery
25
+ ##
26
+ # # View
27
+ #
28
+ # A view is a virtual table defined by a SQL query. You can query views in
29
+ # the browser tool, or by using a query job.
30
+ #
31
+ # BigQuery's views are logical views, not materialized views, which means
32
+ # that the query that defines the view is re-executed every time the view
33
+ # is queried. Queries are billed according to the total amount of data in
34
+ # all table fields referenced directly or indirectly by the top-level
35
+ # query.
36
+ #
37
+ # @example
38
+ # require "google/cloud"
39
+ #
40
+ # gcloud = Google::Cloud.new
41
+ # bigquery = gcloud.bigquery
42
+ # dataset = bigquery.dataset "my_dataset"
43
+ # view = dataset.create_view "my_view",
44
+ # "SELECT name, age FROM [proj:dataset.users]"
45
+ #
46
+ class View
47
+ ##
48
+ # @private The Service object.
49
+ attr_accessor :service
50
+
51
+ ##
52
+ # @private The Google API Client object.
53
+ attr_accessor :gapi
54
+
55
+ ##
56
+ # @private Create an empty Table object.
57
+ def initialize
58
+ @service = nil
59
+ @gapi = Google::Apis::BigqueryV2::Table.new
60
+ end
61
+
62
+ ##
63
+ # A unique ID for this table.
64
+ # The ID must contain only letters (a-z, A-Z), numbers (0-9),
65
+ # or underscores (_). The maximum length is 1,024 characters.
66
+ #
67
+ # @!group Attributes
68
+ #
69
+ def table_id
70
+ @gapi.table_reference.table_id
71
+ end
72
+
73
+ ##
74
+ # The ID of the `Dataset` containing this table.
75
+ #
76
+ # @!group Attributes
77
+ #
78
+ def dataset_id
79
+ @gapi.table_reference.dataset_id
80
+ end
81
+
82
+ ##
83
+ # The ID of the `Project` containing this table.
84
+ #
85
+ # @!group Attributes
86
+ #
87
+ def project_id
88
+ @gapi.table_reference.project_id
89
+ end
90
+
91
+ ##
92
+ # @private The gapi fragment containing the Project ID, Dataset ID, and
93
+ # Table ID as a camel-cased hash.
94
+ def table_ref
95
+ table_ref = @gapi.table_reference
96
+ table_ref = table_ref.to_hash if table_ref.respond_to? :to_hash
97
+ table_ref
98
+ end
99
+
100
+ ##
101
+ # The combined Project ID, Dataset ID, and Table ID for this table, in
102
+ # the format specified by the [Query
103
+ # Reference](https://cloud.google.com/bigquery/query-reference#from):
104
+ # `project_name:datasetId.tableId`. To use this value in queries see
105
+ # {#query_id}.
106
+ #
107
+ # @!group Attributes
108
+ #
109
+ def id
110
+ @gapi.id
111
+ end
112
+
113
+ ##
114
+ # The value returned by {#id}, wrapped in square brackets if the Project
115
+ # ID contains dashes, as specified by the [Query
116
+ # Reference](https://cloud.google.com/bigquery/query-reference#from).
117
+ # Useful in queries.
118
+ #
119
+ # @example
120
+ # require "google/cloud"
121
+ #
122
+ # gcloud = Google::Cloud.new
123
+ # bigquery = gcloud.bigquery
124
+ # dataset = bigquery.dataset "my_dataset"
125
+ # table = dataset.table "my_table"
126
+ #
127
+ # data = bigquery.query "SELECT name FROM #{table.query_id}"
128
+ #
129
+ # @!group Attributes
130
+ #
131
+ def query_id
132
+ project_id["-"] ? "[#{id}]" : id
133
+ end
134
+
135
+ ##
136
+ # The name of the table.
137
+ #
138
+ # @!group Attributes
139
+ #
140
+ def name
141
+ @gapi.friendly_name
142
+ end
143
+
144
+ ##
145
+ # Updates the name of the table.
146
+ #
147
+ # @!group Attributes
148
+ #
149
+ def name= new_name
150
+ @gapi.update! friendly_name: new_name
151
+ patch_gapi! :friendly_name
152
+ end
153
+
154
+ ##
155
+ # A string hash of the dataset.
156
+ #
157
+ # @!group Attributes
158
+ #
159
+ def etag
160
+ ensure_full_data!
161
+ @gapi.etag
162
+ end
163
+
164
+ ##
165
+ # A URL that can be used to access the dataset using the REST API.
166
+ #
167
+ # @!group Attributes
168
+ #
169
+ def api_url
170
+ ensure_full_data!
171
+ @gapi.self_link
172
+ end
173
+
174
+ ##
175
+ # The description of the table.
176
+ #
177
+ # @!group Attributes
178
+ #
179
+ def description
180
+ ensure_full_data!
181
+ @gapi.description
182
+ end
183
+
184
+ ##
185
+ # Updates the description of the table.
186
+ #
187
+ # @!group Attributes
188
+ #
189
+ def description= new_description
190
+ @gapi.update! description: new_description
191
+ patch_gapi! :description
192
+ end
193
+
194
+ ##
195
+ # The time when this table was created.
196
+ #
197
+ # @!group Attributes
198
+ #
199
+ def created_at
200
+ ensure_full_data!
201
+ begin
202
+ Time.at(Integer(@gapi.creation_time) / 1000.0)
203
+ rescue
204
+ nil
205
+ end
206
+ end
207
+
208
+ ##
209
+ # The time when this table expires.
210
+ # If not present, the table will persist indefinitely.
211
+ # Expired tables will be deleted and their storage reclaimed.
212
+ #
213
+ # @!group Attributes
214
+ #
215
+ def expires_at
216
+ ensure_full_data!
217
+ begin
218
+ Time.at(Integer(@gapi.expiration_time) / 1000.0)
219
+ rescue
220
+ nil
221
+ end
222
+ end
223
+
224
+ ##
225
+ # The date when this table was last modified.
226
+ #
227
+ # @!group Attributes
228
+ #
229
+ def modified_at
230
+ ensure_full_data!
231
+ begin
232
+ Time.at(Integer(@gapi.last_modified_time) / 1000.0)
233
+ rescue
234
+ nil
235
+ end
236
+ end
237
+
238
+ ##
239
+ # Checks if the table's type is "TABLE".
240
+ #
241
+ # @!group Attributes
242
+ #
243
+ def table?
244
+ @gapi.type == "TABLE"
245
+ end
246
+
247
+ ##
248
+ # Checks if the table's type is "VIEW".
249
+ #
250
+ # @!group Attributes
251
+ #
252
+ def view?
253
+ @gapi.type == "VIEW"
254
+ end
255
+
256
+ ##
257
+ # The geographic location where the table should reside. Possible
258
+ # values include EU and US. The default value is US.
259
+ #
260
+ # @!group Attributes
261
+ #
262
+ def location
263
+ ensure_full_data!
264
+ @gapi.location
265
+ end
266
+
267
+ ##
268
+ # The schema of the view.
269
+ #
270
+ # @!group Attributes
271
+ #
272
+ def schema
273
+ ensure_full_data!
274
+ Schema.from_gapi(@gapi.schema).freeze
275
+ end
276
+
277
+ ##
278
+ # The fields of the view.
279
+ #
280
+ # @!group Attributes
281
+ #
282
+ def fields
283
+ schema.fields
284
+ end
285
+
286
+ ##
287
+ # The names of the columns in the view.
288
+ #
289
+ # @!group Attributes
290
+ #
291
+ def headers
292
+ fields.map(&:name)
293
+ end
294
+
295
+ ##
296
+ # The query that executes each time the view is loaded.
297
+ #
298
+ # @!group Attributes
299
+ #
300
+ def query
301
+ @gapi.view.query if @gapi.view
302
+ end
303
+
304
+ ##
305
+ # Updates the query that executes each time the view is loaded.
306
+ #
307
+ # @see https://cloud.google.com/bigquery/query-reference BigQuery Query
308
+ # Reference
309
+ #
310
+ # @param [String] new_query The query that defines the view.
311
+ #
312
+ # @example
313
+ # require "google/cloud"
314
+ #
315
+ # gcloud = Google::Cloud.new
316
+ # bigquery = gcloud.bigquery
317
+ # dataset = bigquery.dataset "my_dataset"
318
+ # view = dataset.table "my_view"
319
+ #
320
+ # view.query = "SELECT first_name FROM " \
321
+ # "[my_project:my_dataset.my_table]"
322
+ #
323
+ # @!group Lifecycle
324
+ #
325
+ def query= new_query
326
+ @gapi.view ||= Google::Apis::BigqueryV2::ViewDefinition.new
327
+ @gapi.view.update! query: new_query
328
+ patch_view_gapi! :query
329
+ end
330
+
331
+ ##
332
+ # Runs a query to retrieve all data from the view.
333
+ #
334
+ # @param [Integer] max The maximum number of rows of data to return per
335
+ # page of results. Setting this flag to a small value such as 1000 and
336
+ # then paging through results might improve reliability when the query
337
+ # result set is large. In addition to this limit, responses are also
338
+ # limited to 10 MB. By default, there is no maximum row count, and
339
+ # only the byte limit applies.
340
+ # @param [Integer] timeout How long to wait for the query to complete,
341
+ # in milliseconds, before the request times out and returns. Note that
342
+ # this is only a timeout for the request, not the query. If the query
343
+ # takes longer to run than the timeout value, the call returns without
344
+ # any results and with QueryData#complete? set to false. The default
345
+ # value is 10000 milliseconds (10 seconds).
346
+ # @param [Boolean] cache Whether to look for the result in the query
347
+ # cache. The query cache is a best-effort cache that will be flushed
348
+ # whenever tables in the query are modified. The default value is
349
+ # true. For more information, see [query
350
+ # caching](https://developers.google.com/bigquery/querying-data).
351
+ # @param [Boolean] dryrun If set to `true`, BigQuery doesn't run the
352
+ # job. Instead, if the query is valid, BigQuery returns statistics
353
+ # about the job such as how many bytes would be processed. If the
354
+ # query is invalid, an error returns. The default value is `false`.
355
+ #
356
+ # @return [Google::Cloud::Bigquery::QueryData]
357
+ #
358
+ # @example
359
+ # require "google/cloud"
360
+ #
361
+ # gcloud = Google::Cloud.new
362
+ # bigquery = gcloud.bigquery
363
+ # dataset = bigquery.dataset "my_dataset"
364
+ # view = dataset.table "my_view"
365
+ #
366
+ # data = view.data
367
+ # data.each do |row|
368
+ # puts row["first_name"]
369
+ # end
370
+ # more_data = data.next if data.next?
371
+ #
372
+ # @!group Data
373
+ #
374
+ def data max: nil, timeout: 10000, cache: true, dryrun: nil
375
+ sql = "SELECT * FROM #{query_id}"
376
+ ensure_service!
377
+ options = { max: max, timeout: timeout, cache: cache, dryrun: dryrun }
378
+ gapi = service.query sql, options
379
+ QueryData.from_gapi gapi, service
380
+ end
381
+
382
+ ##
383
+ # Permanently deletes the table.
384
+ #
385
+ # @return [Boolean] Returns `true` if the table was deleted.
386
+ #
387
+ # @example
388
+ # require "google/cloud"
389
+ #
390
+ # gcloud = Google::Cloud.new
391
+ # bigquery = gcloud.bigquery
392
+ # dataset = bigquery.dataset "my_dataset"
393
+ # table = dataset.table "my_table"
394
+ #
395
+ # table.delete
396
+ #
397
+ # @!group Lifecycle
398
+ #
399
+ def delete
400
+ ensure_service!
401
+ service.delete_table dataset_id, table_id
402
+ true
403
+ end
404
+
405
+ ##
406
+ # Reloads the table with current data from the BigQuery service.
407
+ #
408
+ # @!group Lifecycle
409
+ #
410
+ def reload!
411
+ ensure_service!
412
+ gapi = service.get_table dataset_id, table_id
413
+ @gapi = gapi
414
+ end
415
+ alias_method :refresh!, :reload!
416
+
417
+ ##
418
+ # @private New Table from a Google API Client object.
419
+ def self.from_gapi gapi, conn
420
+ new.tap do |f|
421
+ f.gapi = gapi
422
+ f.service = conn
423
+ end
424
+ end
425
+
426
+ protected
427
+
428
+ ##
429
+ # Raise an error unless an active service is available.
430
+ def ensure_service!
431
+ fail "Must have active connection" unless service
432
+ end
433
+
434
+ def patch_gapi! *attributes
435
+ return if attributes.empty?
436
+ patch_args = Hash[attributes.map do |attr|
437
+ [attr, @gapi.send(attr)]
438
+ end]
439
+ patch_table_gapi patch_args
440
+ end
441
+
442
+ def patch_view_gapi! *attributes
443
+ return if attributes.empty?
444
+ patch_args = Hash[attributes.map do |attr|
445
+ [attr, @gapi.view.send(attr)]
446
+ end]
447
+ patch_view_args = Google::Apis::BigqueryV2::ViewDefinition.new(
448
+ patch_args
449
+ )
450
+ patch_table_gapi view: patch_view_args
451
+ end
452
+
453
+ def patch_table_gapi patch_args
454
+ ensure_service!
455
+ patch_gapi = Google::Apis::BigqueryV2::Table.new patch_args
456
+ @gapi = service.patch_table dataset_id, table_id, patch_gapi
457
+ end
458
+
459
+ ##
460
+ # Load the complete representation of the table if it has been
461
+ # only partially loaded by a request to the API list method.
462
+ def ensure_full_data!
463
+ reload_gapi! unless data_complete?
464
+ end
465
+
466
+ def reload_gapi!
467
+ ensure_service!
468
+ gapi = service.get_table dataset_id, table_id
469
+ @gapi = gapi
470
+ end
471
+
472
+ def data_complete?
473
+ @gapi.is_a? Google::Apis::BigqueryV2::Table
474
+ end
475
+ end
476
+ end
477
+ end
478
+ end