google-cloud-bigquery 0.29.0 → 0.30.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -185,8 +185,8 @@ module Google
185
185
  ##
186
186
  # Add reader access to a view.
187
187
  #
188
- # @param [Google::Cloud::Bigquery::View, String] view A view object or
189
- # a string identifier as specified by the [Query
188
+ # @param [Google::Cloud::Bigquery::Table, String] view A table object
189
+ # or a string identifier as specified by the [Query
190
190
  # Reference](https://cloud.google.com/bigquery/query-reference#from):
191
191
  # `project_name:datasetId.tableId`.
192
192
  #
@@ -444,8 +444,8 @@ module Google
444
444
  ##
445
445
  # Remove reader access from a view.
446
446
  #
447
- # @param [Google::Cloud::Bigquery::View, String] view A view object or
448
- # a string identifier as specified by the [Query
447
+ # @param [Google::Cloud::Bigquery::Table, String] view A table object
448
+ # or a string identifier as specified by the [Query
449
449
  # Reference](https://cloud.google.com/bigquery/query-reference#from):
450
450
  # `project_name:datasetId.tableId`.
451
451
  #
@@ -699,8 +699,8 @@ module Google
699
699
  ##
700
700
  # Checks reader access for a view.
701
701
  #
702
- # @param [Google::Cloud::Bigquery::View, String] view A view object or
703
- # a string identifier as specified by the [Query
702
+ # @param [Google::Cloud::Bigquery::Table, String] view A table object
703
+ # or a string identifier as specified by the [Query
704
704
  # Reference](https://cloud.google.com/bigquery/query-reference#from):
705
705
  # `project_name:datasetId.tableId`.
706
706
  #
@@ -31,7 +31,7 @@ module Google
31
31
  #
32
32
  # A job instance is created when you call {Project#query_job},
33
33
  # {Dataset#query_job}, {Table#copy_job}, {Table#extract_job},
34
- # {Table#load_job}, or {View#data}.
34
+ # {Table#load_job}.
35
35
  #
36
36
  # @see https://cloud.google.com/bigquery/docs/managing-jobs Running and
37
37
  # Managing Jobs
@@ -74,19 +74,20 @@ module Google
74
74
  # require "google/cloud/bigquery"
75
75
  #
76
76
  # bigquery = Google::Cloud::Bigquery.new(
77
- # project: "my-project-id",
78
- # keyfile: "/path/to/keyfile.json"
77
+ # project_id: "my-project",
78
+ # credentials: "/path/to/keyfile.json"
79
79
  # )
80
80
  #
81
- # bigquery.project #=> "my-project-id"
81
+ # bigquery.project_id #=> "my-project"
82
82
  #
83
- def project
83
+ def project_id
84
84
  service.project
85
85
  end
86
+ alias_method :project, :project_id
86
87
 
87
88
  ##
88
- # @private Default project.
89
- def self.default_project
89
+ # @private Default project_id.
90
+ def self.default_project_id
90
91
  ENV["BIGQUERY_PROJECT"] ||
91
92
  ENV["GOOGLE_CLOUD_PROJECT"] ||
92
93
  ENV["GCLOUD_PROJECT"] ||
@@ -580,6 +581,10 @@ module Google
580
581
  # Retrieves an existing dataset by ID.
581
582
  #
582
583
  # @param [String] dataset_id The ID of a dataset.
584
+ # @param [Boolean] skip_lookup Optionally create just a local reference
585
+ # object without verifying that the resource exists on the BigQuery
586
+ # service. Calls made on this object will raise errors if the resource
587
+ # does not exist. Default is `false`. Optional.
583
588
  #
584
589
  # @return [Google::Cloud::Bigquery::Dataset, nil] Returns `nil` if the
585
590
  # dataset does not exist.
@@ -592,8 +597,18 @@ module Google
592
597
  # dataset = bigquery.dataset "my_dataset"
593
598
  # puts dataset.name
594
599
  #
595
- def dataset dataset_id
600
+ # @example Avoid retrieving the dataset resource with `skip_lookup`:
601
+ # require "google/cloud/bigquery"
602
+ #
603
+ # bigquery = Google::Cloud::Bigquery.new
604
+ #
605
+ # dataset = bigquery.dataset "my_dataset", skip_lookup: true
606
+ #
607
+ def dataset dataset_id, skip_lookup: nil
596
608
  ensure_service!
609
+ if skip_lookup
610
+ return Dataset.new_reference project, dataset_id, service
611
+ end
597
612
  gapi = service.get_dataset dataset_id
598
613
  Dataset.from_gapi gapi, service
599
614
  rescue Google::Cloud::NotFoundError
@@ -24,7 +24,7 @@ module Google
24
24
  #
25
25
  # A {Job} subclass representing a query operation that may be performed
26
26
  # on a {Table}. A QueryJob instance is created when you call
27
- # {Project#query_job}, {Dataset#query_job}, or {View#data}.
27
+ # {Project#query_job}, {Dataset#query_job}.
28
28
  #
29
29
  # @see https://cloud.google.com/bigquery/querying-data Querying Data
30
30
  # @see https://cloud.google.com/bigquery/docs/reference/v2/jobs Jobs API
@@ -305,9 +305,11 @@ module Google
305
305
  ensure_schema!
306
306
 
307
307
  options = { token: token, max: max, start: start }
308
- data_gapi = service.list_tabledata destination_table_dataset_id,
309
- destination_table_table_id, options
310
- Data.from_gapi data_gapi, destination_table_gapi, service
308
+ data_hash = service.list_tabledata \
309
+ destination_table_dataset_id,
310
+ destination_table_table_id,
311
+ options
312
+ Data.from_gapi_json data_hash, destination_table_gapi, service
311
313
  end
312
314
  alias_method :query_results, :data
313
315
 
@@ -46,7 +46,6 @@ module Google
46
46
  def initialize project, credentials, retries: nil, timeout: nil
47
47
  @project = project
48
48
  @credentials = credentials
49
- @credentials = credentials
50
49
  @retries = retries
51
50
  @timeout = timeout
52
51
  end
@@ -191,10 +190,13 @@ module Google
191
190
  def list_tabledata dataset_id, table_id, options = {}
192
191
  # The list operation is considered idempotent
193
192
  execute backoff: true do
194
- service.list_table_data @project, dataset_id, table_id,
195
- max_results: options.delete(:max),
196
- page_token: options.delete(:token),
197
- start_index: options.delete(:start)
193
+ json_txt = service.list_table_data \
194
+ @project, dataset_id, table_id,
195
+ max_results: options.delete(:max),
196
+ page_token: options.delete(:token),
197
+ start_index: options.delete(:start),
198
+ options: { skip_deserialization: true }
199
+ JSON.parse json_txt, symbolize_names: true
198
200
  end
199
201
  end
200
202
 
@@ -15,7 +15,6 @@
15
15
 
16
16
  require "google/cloud/errors"
17
17
  require "google/cloud/bigquery/service"
18
- require "google/cloud/bigquery/view"
19
18
  require "google/cloud/bigquery/data"
20
19
  require "google/cloud/bigquery/table/list"
21
20
  require "google/cloud/bigquery/schema"
@@ -34,6 +33,15 @@ module Google
34
33
  # records. Every table is defined by a schema that may contain nested and
35
34
  # repeated fields.
36
35
  #
36
+ # The Table class can also represent a
37
+ # [view](https://cloud.google.com/bigquery/docs/views), which is a virtual
38
+ # table defined by a SQL query. BigQuery's views are logical views, not
39
+ # materialized views, which means that the query that defines the view is
40
+ # re-executed every time the view is queried. Queries are billed according
41
+ # to the total amount of data in all table fields referenced directly or
42
+ # indirectly by the top-level query. (See {#view?}, {#query}, {#query=},
43
+ # and {Dataset#create_view}.)
44
+ #
37
45
  # @see https://cloud.google.com/bigquery/preparing-data-for-bigquery
38
46
  # Preparing Data for BigQuery
39
47
  #
@@ -66,6 +74,15 @@ module Google
66
74
  # }
67
75
  # table.insert row
68
76
  #
77
+ # @example Creating a BigQuery view:
78
+ # require "google/cloud/bigquery"
79
+ #
80
+ # bigquery = Google::Cloud::Bigquery.new
81
+ # dataset = bigquery.dataset "my_dataset"
82
+ # view = dataset.create_view "my_view",
83
+ # "SELECT name, age FROM `my_project.my_dataset.my_table`"
84
+ # view.view? # true
85
+ #
69
86
  class Table
70
87
  ##
71
88
  # @private The Service object.
@@ -75,11 +92,16 @@ module Google
75
92
  # @private The Google API Client object.
76
93
  attr_accessor :gapi
77
94
 
95
+ ##
96
+ # @private A Google API Client Table Reference object.
97
+ attr_reader :reference
98
+
78
99
  ##
79
100
  # @private Create an empty Table object.
80
101
  def initialize
81
102
  @service = nil
82
- @gapi = Google::Apis::BigqueryV2::Table.new
103
+ @gapi = nil
104
+ @reference = nil
83
105
  end
84
106
 
85
107
  ##
@@ -91,6 +113,7 @@ module Google
91
113
  # @!group Attributes
92
114
  #
93
115
  def table_id
116
+ return reference.table_id if reference?
94
117
  @gapi.table_reference.table_id
95
118
  end
96
119
 
@@ -103,6 +126,7 @@ module Google
103
126
  # @!group Attributes
104
127
  #
105
128
  def dataset_id
129
+ return reference.dataset_id if reference?
106
130
  @gapi.table_reference.dataset_id
107
131
  end
108
132
 
@@ -114,28 +138,32 @@ module Google
114
138
  # @!group Attributes
115
139
  #
116
140
  def project_id
141
+ return reference.project_id if reference?
117
142
  @gapi.table_reference.project_id
118
143
  end
119
144
 
120
145
  ##
121
146
  # @private The gapi fragment containing the Project ID, Dataset ID, and
122
- # Table ID as a camel-cased hash.
147
+ # Table ID.
148
+ #
149
+ # @return [Google::Apis::BigqueryV2::TableReference]
150
+ #
123
151
  def table_ref
124
- table_ref = @gapi.table_reference
125
- table_ref = table_ref.to_hash if table_ref.respond_to? :to_hash
126
- table_ref
152
+ reference? ? reference : @gapi.table_reference
127
153
  end
128
154
 
129
155
  ###
130
156
  # Checks if the table is time-partitioned. See [Partitioned
131
157
  # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
132
158
  #
133
- # @return [Boolean] `true` when the table is time-partitioned, `false`
134
- # otherwise.
159
+ # @return [Boolean, nil] `true` when the table is time-partitioned, or
160
+ # `false` otherwise, if the object is a resource (see {#resource?});
161
+ # `nil` if the object is a reference (see {#reference?}).
135
162
  #
136
163
  # @!group Attributes
137
164
  #
138
165
  def time_partitioning?
166
+ return nil if reference?
139
167
  !@gapi.time_partitioning.nil?
140
168
  end
141
169
 
@@ -144,11 +172,13 @@ module Google
144
172
  # [Partitioned Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
145
173
  #
146
174
  # @return [String, nil] The partition type. Currently the only supported
147
- # value is "DAY".
175
+ # value is "DAY", or `nil` if the object is a reference (see
176
+ # {#reference?}).
148
177
  #
149
178
  # @!group Attributes
150
179
  #
151
180
  def time_partitioning_type
181
+ return nil if reference?
152
182
  ensure_full_data!
153
183
  @gapi.time_partitioning.type if time_partitioning?
154
184
  end
@@ -161,6 +191,10 @@ module Google
161
191
  # the example below. BigQuery does not allow you to change partitioning
162
192
  # on an existing table.
163
193
  #
194
+ # If the table is not a full resource representation (see
195
+ # {#resource_full?}), the full representation will be retrieved before
196
+ # the update to comply with ETag-based optimistic concurrency control.
197
+ #
164
198
  # @param [String] type The partition type. Currently the only
165
199
  # supported value is "DAY".
166
200
  #
@@ -176,6 +210,7 @@ module Google
176
210
  # @!group Attributes
177
211
  #
178
212
  def time_partitioning_type= type
213
+ reload! unless resource_full?
179
214
  @gapi.time_partitioning ||=
180
215
  Google::Apis::BigqueryV2::TimePartitioning.new
181
216
  @gapi.time_partitioning.type = type
@@ -188,11 +223,13 @@ module Google
188
223
  # [Partitioned Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
189
224
  #
190
225
  # @return [Integer, nil] The expiration time, in seconds, for data in
191
- # partitions.
226
+ # partitions, or `nil` if not present or the object is a reference
227
+ # (see {#reference?}).
192
228
  #
193
229
  # @!group Attributes
194
230
  #
195
231
  def time_partitioning_expiration
232
+ return nil if reference?
196
233
  ensure_full_data!
197
234
  @gapi.time_partitioning.expiration_ms / 1_000 if
198
235
  time_partitioning? &&
@@ -206,6 +243,10 @@ module Google
206
243
  #
207
244
  # See {Table#time_partitioning_type=}.
208
245
  #
246
+ # If the table is not a full resource representation (see
247
+ # {#resource_full?}), the full representation will be retrieved before
248
+ # the update to comply with ETag-based optimistic concurrency control.
249
+ #
209
250
  # @param [Integer] expiration An expiration time, in seconds,
210
251
  # for data in partitions.
211
252
  #
@@ -222,6 +263,7 @@ module Google
222
263
  # @!group Attributes
223
264
  #
224
265
  def time_partitioning_expiration= expiration
266
+ reload! unless resource_full?
225
267
  @gapi.time_partitioning ||=
226
268
  Google::Apis::BigqueryV2::TimePartitioning.new
227
269
  @gapi.time_partitioning.expiration_ms = expiration * 1000
@@ -235,11 +277,13 @@ module Google
235
277
  # `project_name:datasetId.tableId`. To use this value in queries see
236
278
  # {#query_id}.
237
279
  #
238
- # @return [String] The combined ID.
280
+ # @return [String, nil] The combined ID, or `nil` if the object is a
281
+ # reference (see {#reference?}).
239
282
  #
240
283
  # @!group Attributes
241
284
  #
242
285
  def id
286
+ return nil if reference?
243
287
  @gapi.id
244
288
  end
245
289
 
@@ -274,7 +318,7 @@ module Google
274
318
  #
275
319
  def query_id standard_sql: nil, legacy_sql: nil
276
320
  if Convert.resolve_legacy_sql standard_sql, legacy_sql
277
- "[#{id}]"
321
+ "[#{project_id}:#{dataset_id}.#{table_id}]"
278
322
  else
279
323
  "`#{project_id}.#{dataset_id}.#{table_id}`"
280
324
  end
@@ -283,22 +327,29 @@ module Google
283
327
  ##
284
328
  # The name of the table.
285
329
  #
286
- # @return [String] The friendly name.
330
+ # @return [String, nil] The friendly name, or `nil` if the object is a
331
+ # reference (see {#reference?}).
287
332
  #
288
333
  # @!group Attributes
289
334
  #
290
335
  def name
336
+ return nil if reference?
291
337
  @gapi.friendly_name
292
338
  end
293
339
 
294
340
  ##
295
341
  # Updates the name of the table.
296
342
  #
343
+ # If the table is not a full resource representation (see
344
+ # {#resource_full?}), the full representation will be retrieved before
345
+ # the update to comply with ETag-based optimistic concurrency control.
346
+ #
297
347
  # @param [String] new_name The new friendly name.
298
348
  #
299
349
  # @!group Attributes
300
350
  #
301
351
  def name= new_name
352
+ reload! unless resource_full?
302
353
  @gapi.update! friendly_name: new_name
303
354
  patch_gapi! :friendly_name
304
355
  end
@@ -306,11 +357,13 @@ module Google
306
357
  ##
307
358
  # The ETag hash of the table.
308
359
  #
309
- # @return [String] The ETag hash.
360
+ # @return [String, nil] The ETag hash, or `nil` if the object is a
361
+ # reference (see {#reference?}).
310
362
  #
311
363
  # @!group Attributes
312
364
  #
313
365
  def etag
366
+ return nil if reference?
314
367
  ensure_full_data!
315
368
  @gapi.etag
316
369
  end
@@ -318,11 +371,13 @@ module Google
318
371
  ##
319
372
  # A URL that can be used to access the table using the REST API.
320
373
  #
321
- # @return [String] A REST URL for the resource.
374
+ # @return [String, nil] A REST URL for the resource, or `nil` if the
375
+ # object is a reference (see {#reference?}).
322
376
  #
323
377
  # @!group Attributes
324
378
  #
325
379
  def api_url
380
+ return nil if reference?
326
381
  ensure_full_data!
327
382
  @gapi.self_link
328
383
  end
@@ -330,11 +385,13 @@ module Google
330
385
  ##
331
386
  # A user-friendly description of the table.
332
387
  #
333
- # @return [String] The description.
388
+ # @return [String, nil] The description, or `nil` if the object is a
389
+ # reference (see {#reference?}).
334
390
  #
335
391
  # @!group Attributes
336
392
  #
337
393
  def description
394
+ return nil if reference?
338
395
  ensure_full_data!
339
396
  @gapi.description
340
397
  end
@@ -342,11 +399,16 @@ module Google
342
399
  ##
343
400
  # Updates the user-friendly description of the table.
344
401
  #
402
+ # If the table is not a full resource representation (see
403
+ # {#resource_full?}), the full representation will be retrieved before
404
+ # the update to comply with ETag-based optimistic concurrency control.
405
+ #
345
406
  # @param [String] new_description The new user-friendly description.
346
407
  #
347
408
  # @!group Attributes
348
409
  #
349
410
  def description= new_description
411
+ reload! unless resource_full?
350
412
  @gapi.update! description: new_description
351
413
  patch_gapi! :description
352
414
  end
@@ -354,11 +416,13 @@ module Google
354
416
  ##
355
417
  # The number of bytes in the table.
356
418
  #
357
- # @return [Integer] The count of bytes in the table.
419
+ # @return [Integer, nil] The count of bytes in the table, or `nil` if
420
+ # the object is a reference (see {#reference?}).
358
421
  #
359
422
  # @!group Data
360
423
  #
361
424
  def bytes_count
425
+ return nil if reference?
362
426
  ensure_full_data!
363
427
  begin
364
428
  Integer @gapi.num_bytes
@@ -370,11 +434,13 @@ module Google
370
434
  ##
371
435
  # The number of rows in the table.
372
436
  #
373
- # @return [Integer] The count of rows in the table.
437
+ # @return [Integer, nil] The count of rows in the table, or `nil` if the
438
+ # object is a reference (see {#reference?}).
374
439
  #
375
440
  # @!group Data
376
441
  #
377
442
  def rows_count
443
+ return nil if reference?
378
444
  ensure_full_data!
379
445
  begin
380
446
  Integer @gapi.num_rows
@@ -386,11 +452,13 @@ module Google
386
452
  ##
387
453
  # The time when this table was created.
388
454
  #
389
- # @return [Time, nil] The creation time.
455
+ # @return [Time, nil] The creation time, or `nil` if the object is a
456
+ # reference (see {#reference?}).
390
457
  #
391
458
  # @!group Attributes
392
459
  #
393
460
  def created_at
461
+ return nil if reference?
394
462
  ensure_full_data!
395
463
  begin
396
464
  ::Time.at(Integer(@gapi.creation_time) / 1000.0)
@@ -404,11 +472,13 @@ module Google
404
472
  # If not present, the table will persist indefinitely.
405
473
  # Expired tables will be deleted and their storage reclaimed.
406
474
  #
407
- # @return [Time, nil] The expiration time.
475
+ # @return [Time, nil] The expiration time, or `nil` if not present or
476
+ # the object is a reference (see {#reference?}).
408
477
  #
409
478
  # @!group Attributes
410
479
  #
411
480
  def expires_at
481
+ return nil if reference?
412
482
  ensure_full_data!
413
483
  begin
414
484
  ::Time.at(Integer(@gapi.expiration_time) / 1000.0)
@@ -420,11 +490,13 @@ module Google
420
490
  ##
421
491
  # The date when this table was last modified.
422
492
  #
423
- # @return [Time, nil] The last modified time.
493
+ # @return [Time, nil] The last modified time, or `nil` if not present or
494
+ # the object is a reference (see {#reference?}).
424
495
  #
425
496
  # @!group Attributes
426
497
  #
427
498
  def modified_at
499
+ return nil if reference?
428
500
  ensure_full_data!
429
501
  begin
430
502
  ::Time.at(Integer(@gapi.last_modified_time) / 1000.0)
@@ -436,34 +508,45 @@ module Google
436
508
  ##
437
509
  # Checks if the table's type is "TABLE".
438
510
  #
439
- # @return [Boolean] `true` when the type is `TABLE`, `false` otherwise.
511
+ # @return [Boolean, nil] `true` when the type is `TABLE`, `false`
512
+ # otherwise, if the object is a resource (see {#resource?}); `nil` if
513
+ # the object is a reference (see {#reference?}).
440
514
  #
441
515
  # @!group Attributes
442
516
  #
443
517
  def table?
518
+ return nil if reference?
444
519
  @gapi.type == "TABLE"
445
520
  end
446
521
 
447
522
  ##
448
- # Checks if the table's type is "VIEW".
523
+ # Checks if the table's type is "VIEW", indicating that the table
524
+ # represents a BigQuery view. See {Dataset#create_view}.
449
525
  #
450
- # @return [Boolean] `true` when the type is `VIEW`, `false` otherwise.
526
+ # @return [Boolean, nil] `true` when the type is `VIEW`, `false`
527
+ # otherwise, if the object is a resource (see {#resource?}); `nil` if
528
+ # the object is a reference (see {#reference?}).
451
529
  #
452
530
  # @!group Attributes
453
531
  #
454
532
  def view?
533
+ return nil if reference?
455
534
  @gapi.type == "VIEW"
456
535
  end
457
536
 
458
537
  ##
459
- # Checks if the table's type is "EXTERNAL".
538
+ # Checks if the table's type is "EXTERNAL", indicating that the table
539
+ # represents an External Data Source. See {#external?} and
540
+ # {External::DataSource}.
460
541
  #
461
- # @return [Boolean] `true` when the type is `EXTERNAL`, `false`
462
- # otherwise.
542
+ # @return [Boolean, nil] `true` when the type is `EXTERNAL`, `false`
543
+ # otherwise, if the object is a resource (see {#resource?}); `nil` if
544
+ # the object is a reference (see {#reference?}).
463
545
  #
464
546
  # @!group Attributes
465
547
  #
466
548
  def external?
549
+ return nil if reference?
467
550
  @gapi.type == "EXTERNAL"
468
551
  end
469
552
 
@@ -471,11 +554,12 @@ module Google
471
554
  # The geographic location where the table should reside. Possible
472
555
  # values include `EU` and `US`. The default value is `US`.
473
556
  #
474
- # @return [String] The location code.
557
+ # @return [String, nil] The location code.
475
558
  #
476
559
  # @!group Attributes
477
560
  #
478
561
  def location
562
+ return nil if reference?
479
563
  ensure_full_data!
480
564
  @gapi.location
481
565
  end
@@ -488,7 +572,7 @@ module Google
488
572
  # The returned hash is frozen and changes are not allowed. Use
489
573
  # {#labels=} to replace the entire hash.
490
574
  #
491
- # @return [Hash<String, String>] A hash containing key/value pairs.
575
+ # @return [Hash<String, String>, nil] A hash containing key/value pairs.
492
576
  #
493
577
  # @example
494
578
  # require "google/cloud/bigquery"
@@ -503,6 +587,7 @@ module Google
503
587
  # @!group Attributes
504
588
  #
505
589
  def labels
590
+ return nil if reference?
506
591
  m = @gapi.labels
507
592
  m = m.to_h if m.respond_to? :to_h
508
593
  m.dup.freeze
@@ -513,6 +598,10 @@ module Google
513
598
  # Labels are used to organize and group tables. See [Using
514
599
  # Labels](https://cloud.google.com/bigquery/docs/labels).
515
600
  #
601
+ # If the table is not a full resource representation (see
602
+ # {#resource_full?}), the full representation will be retrieved before
603
+ # the update to comply with ETag-based optimistic concurrency control.
604
+ #
516
605
  # @param [Hash<String, String>] labels A hash containing key/value
517
606
  # pairs.
518
607
  #
@@ -535,14 +624,18 @@ module Google
535
624
  # @!group Attributes
536
625
  #
537
626
  def labels= labels
627
+ reload! unless resource_full?
538
628
  @gapi.labels = labels
539
629
  patch_gapi! :labels
540
630
  end
541
631
 
542
632
  ##
543
- # Returns the table's schema. This method can also be used to set,
544
- # replace, or add to the schema by passing a block. See {Schema} for
545
- # available methods.
633
+ # Returns the table's schema. If the table is not a view (See {#view?}),
634
+ # this method can also be used to set, replace, or add to the schema by
635
+ # passing a block. See {Schema} for available methods.
636
+ #
637
+ # If the table is not a full resource representation (see
638
+ # {#resource_full?}), the full representation will be retrieved.
546
639
  #
547
640
  # @param [Boolean] replace Whether to replace the existing schema with
548
641
  # the new schema. If `true`, the fields will replace the existing
@@ -552,7 +645,7 @@ module Google
552
645
  # @yield [schema] a block for setting the schema
553
646
  # @yieldparam [Schema] schema the object accepting the schema
554
647
  #
555
- # @return [Google::Cloud::Bigquery::Schema] A frozen schema object.
648
+ # @return [Google::Cloud::Bigquery::Schema, nil] A frozen schema object.
556
649
  #
557
650
  # @example
558
651
  # require "google/cloud/bigquery"
@@ -572,7 +665,8 @@ module Google
572
665
  # @!group Attributes
573
666
  #
574
667
  def schema replace: false
575
- ensure_full_data!
668
+ return nil if reference? && !block_given?
669
+ reload! unless resource_full?
576
670
  schema_builder = Schema.from_gapi @gapi.schema
577
671
  if block_given?
578
672
  schema_builder = Schema.from_gapi if replace
@@ -588,7 +682,7 @@ module Google
588
682
  ##
589
683
  # The fields of the table, obtained from its schema.
590
684
  #
591
- # @return [Array<Schema::Field>] An array of field objects.
685
+ # @return [Array<Schema::Field>, nil] An array of field objects.
592
686
  #
593
687
  # @example
594
688
  # require "google/cloud/bigquery"
@@ -604,13 +698,14 @@ module Google
604
698
  # @!group Attributes
605
699
  #
606
700
  def fields
701
+ return nil if reference?
607
702
  schema.fields
608
703
  end
609
704
 
610
705
  ##
611
706
  # The names of the columns in the table, obtained from its schema.
612
707
  #
613
- # @return [Array<Symbol>] An array of column names.
708
+ # @return [Array<Symbol>, nil] An array of column names.
614
709
  #
615
710
  # @example
616
711
  # require "google/cloud/bigquery"
@@ -626,6 +721,7 @@ module Google
626
721
  # @!group Attributes
627
722
  #
628
723
  def headers
724
+ return nil if reference?
629
725
  schema.headers
630
726
  end
631
727
 
@@ -642,11 +738,13 @@ module Google
642
738
  # @see https://cloud.google.com/bigquery/external-data-sources
643
739
  # Querying External Data Sources
644
740
  #
645
- # @return [External::DataSource] The external data source.
741
+ # @return [External::DataSource, nil] The external data source.
646
742
  #
647
743
  # @!group Attributes
648
744
  #
649
745
  def external
746
+ return nil if reference?
747
+ ensure_full_data!
650
748
  return nil if @gapi.external_data_configuration.nil?
651
749
  External.from_gapi(@gapi.external_data_configuration).freeze
652
750
  end
@@ -661,6 +759,10 @@ module Google
661
759
  # Use only if the table represents an External Data Source. See
662
760
  # {#external?} and {External::DataSource}.
663
761
  #
762
+ # If the table is not a full resource representation (see
763
+ # {#resource_full?}), the full representation will be retrieved before
764
+ # the update to comply with ETag-based optimistic concurrency control.
765
+ #
664
766
  # @see https://cloud.google.com/bigquery/external-data-sources
665
767
  # Querying External Data Sources
666
768
  #
@@ -669,6 +771,7 @@ module Google
669
771
  # @!group Attributes
670
772
  #
671
773
  def external= external
774
+ reload! unless resource_full?
672
775
  @gapi.external_data_configuration = external.to_gapi
673
776
  patch_gapi! :external_data_configuration
674
777
  end
@@ -679,11 +782,14 @@ module Google
679
782
  # if the table is not being streamed to or if there is no data in the
680
783
  # streaming buffer.
681
784
  #
682
- # @return [Integer] The estimated number of bytes in the buffer.
785
+ # @return [Integer, nil] The estimated number of bytes in the buffer, or
786
+ # `nil` if not present or the object is a reference (see
787
+ # {#reference?}).
683
788
  #
684
789
  # @!group Attributes
685
790
  #
686
791
  def buffer_bytes
792
+ return nil if reference?
687
793
  ensure_full_data!
688
794
  @gapi.streaming_buffer.estimated_bytes if @gapi.streaming_buffer
689
795
  end
@@ -694,11 +800,14 @@ module Google
694
800
  # if the table is not being streamed to or if there is no data in the
695
801
  # streaming buffer.
696
802
  #
697
- # @return [Integer] The estimated number of rows in the buffer.
803
+ # @return [Integer, nil] The estimated number of rows in the buffer, or
804
+ # `nil` if not present or the object is a reference (see
805
+ # {#reference?}).
698
806
  #
699
807
  # @!group Attributes
700
808
  #
701
809
  def buffer_rows
810
+ return nil if reference?
702
811
  ensure_full_data!
703
812
  @gapi.streaming_buffer.estimated_rows if @gapi.streaming_buffer
704
813
  end
@@ -708,11 +817,13 @@ module Google
708
817
  # buffer, if one is present. This field will be absent if the table is
709
818
  # not being streamed to or if there is no data in the streaming buffer.
710
819
  #
711
- # @return [Time, nil] The oldest entry time.
820
+ # @return [Time, nil] The oldest entry time, or `nil` if not present or
821
+ # the object is a reference (see {#reference?}).
712
822
  #
713
823
  # @!group Attributes
714
824
  #
715
825
  def buffer_oldest_at
826
+ return nil if reference?
716
827
  ensure_full_data!
717
828
  return nil unless @gapi.streaming_buffer
718
829
  oldest_entry_time = @gapi.streaming_buffer.oldest_entry_time
@@ -723,9 +834,142 @@ module Google
723
834
  end
724
835
  end
725
836
 
837
+ ##
838
+ # The query that executes each time the view is loaded.
839
+ #
840
+ # @return [String] The query that defines the view.
841
+ #
842
+ # @!group Attributes
843
+ #
844
+ def query
845
+ @gapi.view.query if @gapi.view
846
+ end
847
+
848
+ ##
849
+ # Updates the query that executes each time the view is loaded.
850
+ #
851
+ # This sets the query using standard SQL. To specify legacy SQL or to
852
+ # use user-defined function resources use (#set_query) instead.
853
+ #
854
+ # @see https://cloud.google.com/bigquery/query-reference BigQuery Query
855
+ # Reference
856
+ #
857
+ # @param [String] new_query The query that defines the view.
858
+ #
859
+ # @example
860
+ # require "google/cloud/bigquery"
861
+ #
862
+ # bigquery = Google::Cloud::Bigquery.new
863
+ # dataset = bigquery.dataset "my_dataset"
864
+ # view = dataset.table "my_view"
865
+ #
866
+ # view.query = "SELECT first_name FROM " \
867
+ # "`my_project.my_dataset.my_table`"
868
+ #
869
+ # @!group Lifecycle
870
+ #
871
+ def query= new_query
872
+ set_query new_query
873
+ end
874
+
875
+ ##
876
+ # Updates the query that executes each time the view is loaded. Allows
877
+ # setting of standard vs. legacy SQL and user-defined function
878
+ # resources.
879
+ #
880
+ # @see https://cloud.google.com/bigquery/query-reference BigQuery Query
881
+ # Reference
882
+ #
883
+ # @param [String] query The query that defines the view.
884
+ # @param [Boolean] standard_sql Specifies whether to use BigQuery's
885
+ # [standard
886
+ # SQL](https://cloud.google.com/bigquery/docs/reference/standard-sql/)
887
+ # dialect. Optional. The default value is true.
888
+ # @param [Boolean] legacy_sql Specifies whether to use BigQuery's
889
+ # [legacy
890
+ # SQL](https://cloud.google.com/bigquery/docs/reference/legacy-sql)
891
+ # dialect. Optional. The default value is false.
892
+ # @param [Array<String>, String] udfs User-defined function resources
893
+ # used in the query. May be either a code resource to load from a
894
+ # Google Cloud Storage URI (`gs://bucket/path`), or an inline resource
895
+ # that contains code for a user-defined function (UDF). Providing an
896
+ # inline code resource is equivalent to providing a URI for a file
897
+ # containing the same code. See [User-Defined
898
+ # Functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/user-defined-functions).
899
+ #
900
+ # @example
901
+ # require "google/cloud/bigquery"
902
+ #
903
+ # bigquery = Google::Cloud::Bigquery.new
904
+ # dataset = bigquery.dataset "my_dataset"
905
+ # view = dataset.table "my_view"
906
+ #
907
+ # view.set_query "SELECT first_name FROM " \
908
+ # "`my_project.my_dataset.my_table`",
909
+ # standard_sql: true
910
+ #
911
+ # @!group Lifecycle
912
+ #
913
+ def set_query query, standard_sql: nil, legacy_sql: nil, udfs: nil
914
+ @gapi.view = Google::Apis::BigqueryV2::ViewDefinition.new \
915
+ query: query,
916
+ use_legacy_sql: Convert.resolve_legacy_sql(standard_sql,
917
+ legacy_sql),
918
+ user_defined_function_resources: udfs_gapi(udfs)
919
+ patch_gapi! :view
920
+ end
921
+
922
+ ##
923
+ # Checks if the view's query is using legacy sql.
924
+ #
925
+ # @return [Boolean] `true` when legacy sql is used, `false` otherwise.
926
+ #
927
+ # @!group Attributes
928
+ #
929
+ def query_legacy_sql?
930
+ val = @gapi.view.use_legacy_sql
931
+ return true if val.nil?
932
+ val
933
+ end
934
+
935
+ ##
936
+ # Checks if the view's query is using standard sql.
937
+ #
938
+ # @return [Boolean] `true` when standard sql is used, `false` otherwise.
939
+ #
940
+ # @!group Attributes
941
+ #
942
+ def query_standard_sql?
943
+ !query_legacy_sql?
944
+ end
945
+
946
+ ##
947
+ # The user-defined function resources used in the view's query. May be
948
+ # either a code resource to load from a Google Cloud Storage URI
949
+ # (`gs://bucket/path`), or an inline resource that contains code for a
950
+ # user-defined function (UDF). Providing an inline code resource is
951
+ # equivalent to providing a URI for a file containing the same code. See
952
+ # [User-Defined
953
+ # Functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/user-defined-functions).
954
+ #
955
+ # @return [Array<String>] An array containing Google Cloud Storage URIs
956
+ # and/or inline source code.
957
+ #
958
+ # @!group Attributes
959
+ #
960
+ def query_udfs
961
+ udfs_gapi = @gapi.view.user_defined_function_resources
962
+ return [] if udfs_gapi.nil?
963
+ Array(udfs_gapi).map { |udf| udf.inline_code || udf.resource_uri }
964
+ end
965
+
726
966
  ##
727
967
  # Retrieves data from the table.
728
968
  #
969
+ # If the table is not a full resource representation (see
970
+ # {#resource_full?}), the full representation will be retrieved before
971
+ # the data retrieval.
972
+ #
729
973
  # @param [String] token Page token, returned by a previous call,
730
974
  # identifying the result set.
731
975
  #
@@ -765,9 +1009,11 @@ module Google
765
1009
  #
766
1010
  def data token: nil, max: nil, start: nil
767
1011
  ensure_service!
1012
+ reload! unless resource_full?
768
1013
  options = { token: token, max: max, start: start }
769
- data_gapi = service.list_tabledata dataset_id, table_id, options
770
- Data.from_gapi data_gapi, gapi, service
1014
+ data_json = service.list_tabledata \
1015
+ dataset_id, table_id, options
1016
+ Data.from_gapi_json data_json, gapi, service
771
1017
  end
772
1018
 
773
1019
  ##
@@ -1438,6 +1684,19 @@ module Google
1438
1684
  # ]
1439
1685
  # table.insert rows
1440
1686
  #
1687
+ # @example Avoid retrieving the dataset and table with `skip_lookup`:
1688
+ # require "google/cloud/bigquery"
1689
+ #
1690
+ # bigquery = Google::Cloud::Bigquery.new
1691
+ # dataset = bigquery.dataset "my_dataset", skip_lookup: true
1692
+ # table = dataset.table "my_table", skip_lookup: true
1693
+ #
1694
+ # rows = [
1695
+ # { "first_name" => "Alice", "age" => 21 },
1696
+ # { "first_name" => "Bob", "age" => 22 }
1697
+ # ]
1698
+ # table.insert rows
1699
+ #
1441
1700
  # @!group Data
1442
1701
  #
1443
1702
  def insert rows, skip_invalid: nil, ignore_unknown: nil
@@ -1451,7 +1710,7 @@ module Google
1451
1710
  end
1452
1711
 
1453
1712
  ##
1454
- # Create an asynchonous inserter object used to insert rows in batches.
1713
+ # Create an asynchronous inserter object used to insert rows in batches.
1455
1714
  #
1456
1715
  # @param [Boolean] skip_invalid Insert all valid rows of a request, even
1457
1716
  # if invalid rows exist. The default value is `false`, which causes
@@ -1469,8 +1728,8 @@ module Google
1469
1728
  # @attr_reader [Numeric] threads The number of threads used to insert
1470
1729
  # batches of rows. Default is 4.
1471
1730
  # @yield [response] the callback for when a batch of rows is inserted
1472
- # @yieldparam [InsertResponse] response the result of the asynchonous
1473
- # insert
1731
+ # @yieldparam [Table::AsyncInserter::Result] result the result of the
1732
+ # asynchronous insert
1474
1733
  #
1475
1734
  # @return [Table::AsyncInserter] Returns inserter object.
1476
1735
  #
@@ -1480,9 +1739,13 @@ module Google
1480
1739
  # bigquery = Google::Cloud::Bigquery.new
1481
1740
  # dataset = bigquery.dataset "my_dataset"
1482
1741
  # table = dataset.table "my_table"
1483
- # inserter = table.insert_async do |response|
1484
- # log_insert "inserted #{response.insert_count} rows " \
1485
- # "with #{response.error_count} errors"
1742
+ # inserter = table.insert_async do |result|
1743
+ # if result.error?
1744
+ # log_error result.error
1745
+ # else
1746
+ # log_insert "inserted #{result.insert_count} rows " \
1747
+ # "with #{result.error_count} errors"
1748
+ # end
1486
1749
  # end
1487
1750
  #
1488
1751
  # rows = [
@@ -1529,6 +1792,19 @@ module Google
1529
1792
  ##
1530
1793
  # Reloads the table with current data from the BigQuery service.
1531
1794
  #
1795
+ # @return [Google::Cloud::Bigquery::Table] Returns the reloaded
1796
+ # table.
1797
+ #
1798
+ # @example Skip retrieving the table from the service, then load it:
1799
+ # require "google/cloud/bigquery"
1800
+ #
1801
+ # bigquery = Google::Cloud::Bigquery.new
1802
+ #
1803
+ # dataset = bigquery.dataset "my_dataset"
1804
+ # table = dataset.table "my_table", skip_lookup: true
1805
+ #
1806
+ # table.reload!
1807
+ #
1532
1808
  # @!group Lifecycle
1533
1809
  #
1534
1810
  def reload!
@@ -1538,16 +1814,152 @@ module Google
1538
1814
  end
1539
1815
  alias_method :refresh!, :reload!
1540
1816
 
1817
+ ##
1818
+ # Determines whether the table exists in the BigQuery service. The
1819
+ # result is cached locally.
1820
+ #
1821
+ # @return [Boolean] `true` when the table exists in the BigQuery
1822
+ # service, `false` otherwise.
1823
+ #
1824
+ # @example
1825
+ # require "google/cloud/bigquery"
1826
+ #
1827
+ # bigquery = Google::Cloud::Bigquery.new
1828
+ #
1829
+ # dataset = bigquery.dataset "my_dataset"
1830
+ # table = dataset.table "my_table", skip_lookup: true
1831
+ # table.exists? # true
1832
+ #
1833
+ def exists?
1834
+ # Always true if we have a gapi object
1835
+ return true unless reference?
1836
+ # If we have a value, return it
1837
+ return @exists unless @exists.nil?
1838
+ ensure_gapi!
1839
+ @exists = true
1840
+ rescue Google::Cloud::NotFoundError
1841
+ @exists = false
1842
+ end
1843
+
1844
+ ##
1845
+ # Whether the table was created without retrieving the resource
1846
+ # representation from the BigQuery service.
1847
+ #
1848
+ # @return [Boolean] `true` when the table is just a local reference
1849
+ # object, `false` otherwise.
1850
+ #
1851
+ # @example
1852
+ # require "google/cloud/bigquery"
1853
+ #
1854
+ # bigquery = Google::Cloud::Bigquery.new
1855
+ #
1856
+ # dataset = bigquery.dataset "my_dataset"
1857
+ # table = dataset.table "my_table", skip_lookup: true
1858
+ #
1859
+ # table.reference? # true
1860
+ # table.reload!
1861
+ # table.reference? # false
1862
+ #
1863
+ def reference?
1864
+ @gapi.nil?
1865
+ end
1866
+
1867
+ ##
1868
+ # Whether the table was created with a resource representation from
1869
+ # the BigQuery service.
1870
+ #
1871
+ # @return [Boolean] `true` when the table was created with a resource
1872
+ # representation, `false` otherwise.
1873
+ #
1874
+ # @example
1875
+ # require "google/cloud/bigquery"
1876
+ #
1877
+ # bigquery = Google::Cloud::Bigquery.new
1878
+ #
1879
+ # dataset = bigquery.dataset "my_dataset"
1880
+ # table = dataset.table "my_table", skip_lookup: true
1881
+ #
1882
+ # table.resource? # false
1883
+ # table.reload!
1884
+ # table.resource? # true
1885
+ #
1886
+ def resource?
1887
+ !@gapi.nil?
1888
+ end
1889
+
1890
+ ##
1891
+ # Whether the table was created with a partial resource representation
1892
+ # from the BigQuery service by retrieval through {Dataset#tables}.
1893
+ # See [Tables: list
1894
+ # response](https://cloud.google.com/bigquery/docs/reference/rest/v2/tables/list#response)
1895
+ # for the contents of the partial representation. Accessing any
1896
+ # attribute outside of the partial representation will result in loading
1897
+ # the full representation.
1898
+ #
1899
+ # @return [Boolean] `true` when the table was created with a partial
1900
+ # resource representation, `false` otherwise.
1901
+ #
1902
+ # @example
1903
+ # require "google/cloud/bigquery"
1904
+ #
1905
+ # bigquery = Google::Cloud::Bigquery.new
1906
+ #
1907
+ # dataset = bigquery.dataset "my_dataset"
1908
+ # table = dataset.tables.first
1909
+ #
1910
+ # table.resource_partial? # true
1911
+ # table.description # Loads the full resource.
1912
+ # table.resource_partial? # false
1913
+ #
1914
+ def resource_partial?
1915
+ @gapi.is_a? Google::Apis::BigqueryV2::TableList::Table
1916
+ end
1917
+
1918
+ ##
1919
+ # Whether the table was created with a full resource representation
1920
+ # from the BigQuery service.
1921
+ #
1922
+ # @return [Boolean] `true` when the table was created with a full
1923
+ # resource representation, `false` otherwise.
1924
+ #
1925
+ # @example
1926
+ # require "google/cloud/bigquery"
1927
+ #
1928
+ # bigquery = Google::Cloud::Bigquery.new
1929
+ #
1930
+ # dataset = bigquery.dataset "my_dataset"
1931
+ # table = dataset.table "my_table"
1932
+ #
1933
+ # table.resource_full? # true
1934
+ #
1935
+ def resource_full?
1936
+ @gapi.is_a? Google::Apis::BigqueryV2::Table
1937
+ end
1938
+
1541
1939
  ##
1542
1940
  # @private New Table from a Google API Client object.
1543
1941
  def self.from_gapi gapi, conn
1544
- klass = class_for gapi
1545
- klass.new.tap do |f|
1942
+ new.tap do |f|
1546
1943
  f.gapi = gapi
1547
1944
  f.service = conn
1548
1945
  end
1549
1946
  end
1550
1947
 
1948
+ ##
1949
+ # @private New lazy Table object without making an HTTP request.
1950
+ def self.new_reference project_id, dataset_id, table_id, service
1951
+ # TODO: raise if dataset_id or table_id is nil?
1952
+ new.tap do |b|
1953
+ reference_gapi = Google::Apis::BigqueryV2::TableReference.new(
1954
+ project_id: project_id,
1955
+ dataset_id: dataset_id,
1956
+ table_id: table_id
1957
+ )
1958
+ b.service = service
1959
+ b.instance_variable_set :@reference, reference_gapi
1960
+ end
1961
+ end
1962
+
1551
1963
  protected
1552
1964
 
1553
1965
  ##
@@ -1556,6 +1968,15 @@ module Google
1556
1968
  fail "Must have active connection" unless service
1557
1969
  end
1558
1970
 
1971
+ ##
1972
+ # Ensures the Google::Apis::BigqueryV2::Table object has been loaded
1973
+ # from the service.
1974
+ def ensure_gapi!
1975
+ ensure_service!
1976
+ return unless reference?
1977
+ reload!
1978
+ end
1979
+
1559
1980
  def patch_gapi! *attributes
1560
1981
  return if attributes.empty?
1561
1982
  ensure_service!
@@ -1571,11 +1992,6 @@ module Google
1571
1992
  reload!
1572
1993
  end
1573
1994
 
1574
- def self.class_for gapi
1575
- return View if gapi.type == "VIEW"
1576
- self
1577
- end
1578
-
1579
1995
  def load_storage url, options = {}
1580
1996
  # Convert to storage URL
1581
1997
  url = url.to_gs_url if url.respond_to? :to_gs_url
@@ -1608,19 +2024,28 @@ module Google
1608
2024
  # Load the complete representation of the table if it has been
1609
2025
  # only partially loaded by a request to the API list method.
1610
2026
  def ensure_full_data!
1611
- reload_gapi! unless data_complete?
1612
- end
1613
-
1614
- def reload_gapi!
1615
- ensure_service!
1616
- gapi = service.get_table dataset_id, table_id
1617
- @gapi = gapi
2027
+ reload! unless data_complete?
1618
2028
  end
1619
2029
 
1620
2030
  def data_complete?
1621
2031
  @gapi.is_a? Google::Apis::BigqueryV2::Table
1622
2032
  end
1623
2033
 
2034
+ ##
2035
+ # Supports views.
2036
+ def udfs_gapi array_or_str
2037
+ return [] if array_or_str.nil?
2038
+ Array(array_or_str).map do |uri_or_code|
2039
+ resource = Google::Apis::BigqueryV2::UserDefinedFunctionResource.new
2040
+ if uri_or_code.start_with?("gs://")
2041
+ resource.resource_uri = uri_or_code
2042
+ else
2043
+ resource.inline_code = uri_or_code
2044
+ end
2045
+ resource
2046
+ end
2047
+ end
2048
+
1624
2049
  private
1625
2050
 
1626
2051
  def get_table_ref table