google-cloud-bigquery 1.21.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +7 -0
  2. data/.yardopts +16 -0
  3. data/AUTHENTICATION.md +158 -0
  4. data/CHANGELOG.md +397 -0
  5. data/CODE_OF_CONDUCT.md +40 -0
  6. data/CONTRIBUTING.md +188 -0
  7. data/LICENSE +201 -0
  8. data/LOGGING.md +27 -0
  9. data/OVERVIEW.md +463 -0
  10. data/TROUBLESHOOTING.md +31 -0
  11. data/lib/google-cloud-bigquery.rb +139 -0
  12. data/lib/google/cloud/bigquery.rb +145 -0
  13. data/lib/google/cloud/bigquery/argument.rb +197 -0
  14. data/lib/google/cloud/bigquery/convert.rb +383 -0
  15. data/lib/google/cloud/bigquery/copy_job.rb +316 -0
  16. data/lib/google/cloud/bigquery/credentials.rb +50 -0
  17. data/lib/google/cloud/bigquery/data.rb +526 -0
  18. data/lib/google/cloud/bigquery/dataset.rb +2845 -0
  19. data/lib/google/cloud/bigquery/dataset/access.rb +1021 -0
  20. data/lib/google/cloud/bigquery/dataset/list.rb +162 -0
  21. data/lib/google/cloud/bigquery/encryption_configuration.rb +123 -0
  22. data/lib/google/cloud/bigquery/external.rb +2432 -0
  23. data/lib/google/cloud/bigquery/extract_job.rb +368 -0
  24. data/lib/google/cloud/bigquery/insert_response.rb +180 -0
  25. data/lib/google/cloud/bigquery/job.rb +657 -0
  26. data/lib/google/cloud/bigquery/job/list.rb +162 -0
  27. data/lib/google/cloud/bigquery/load_job.rb +1704 -0
  28. data/lib/google/cloud/bigquery/model.rb +740 -0
  29. data/lib/google/cloud/bigquery/model/list.rb +164 -0
  30. data/lib/google/cloud/bigquery/project.rb +1655 -0
  31. data/lib/google/cloud/bigquery/project/list.rb +161 -0
  32. data/lib/google/cloud/bigquery/query_job.rb +1695 -0
  33. data/lib/google/cloud/bigquery/routine.rb +1108 -0
  34. data/lib/google/cloud/bigquery/routine/list.rb +165 -0
  35. data/lib/google/cloud/bigquery/schema.rb +564 -0
  36. data/lib/google/cloud/bigquery/schema/field.rb +668 -0
  37. data/lib/google/cloud/bigquery/service.rb +589 -0
  38. data/lib/google/cloud/bigquery/standard_sql.rb +495 -0
  39. data/lib/google/cloud/bigquery/table.rb +3340 -0
  40. data/lib/google/cloud/bigquery/table/async_inserter.rb +520 -0
  41. data/lib/google/cloud/bigquery/table/list.rb +172 -0
  42. data/lib/google/cloud/bigquery/time.rb +65 -0
  43. data/lib/google/cloud/bigquery/version.rb +22 -0
  44. metadata +297 -0
@@ -0,0 +1,162 @@
1
+ # Copyright 2015 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ require "delegate"
17
+
18
+ module Google
19
+ module Cloud
20
+ module Bigquery
21
+ class Dataset
22
+ ##
23
+ # Dataset::List is a special case Array with additional values.
24
+ class List < DelegateClass(::Array)
25
+ ##
26
+ # If not empty, indicates that there are more records that match
27
+ # the request and this value should be passed to continue.
28
+ attr_accessor :token
29
+
30
+ # A hash of this page of results.
31
+ attr_accessor :etag
32
+
33
+ ##
34
+ # @private Create a new Dataset::List with an array of datasets.
35
+ def initialize arr = []
36
+ super arr
37
+ end
38
+
39
+ ##
40
+ # Whether there is a next page of datasets.
41
+ #
42
+ # @return [Boolean]
43
+ #
44
+ # @example
45
+ # require "google/cloud/bigquery"
46
+ #
47
+ # bigquery = Google::Cloud::Bigquery.new
48
+ #
49
+ # datasets = bigquery.datasets
50
+ # if datasets.next?
51
+ # next_datasets = datasets.next
52
+ # end
53
+ def next?
54
+ !token.nil?
55
+ end
56
+
57
+ ##
58
+ # Retrieve the next page of datasets.
59
+ #
60
+ # @return [Dataset::List]
61
+ #
62
+ # @example
63
+ # require "google/cloud/bigquery"
64
+ #
65
+ # bigquery = Google::Cloud::Bigquery.new
66
+ #
67
+ # datasets = bigquery.datasets
68
+ # if datasets.next?
69
+ # next_datasets = datasets.next
70
+ # end
71
+ def next
72
+ return nil unless next?
73
+ ensure_service!
74
+ gapi = @service.list_datasets all: @hidden, filter: @filter, token: token, max: @max
75
+ self.class.from_gapi gapi, @service, @hidden, @filter, @max
76
+ end
77
+
78
+ ##
79
+ # Retrieves remaining results by repeatedly invoking {#next} until
80
+ # {#next?} returns `false`. Calls the given block once for each
81
+ # result, which is passed as the argument to the block.
82
+ #
83
+ # An Enumerator is returned if no block is given.
84
+ #
85
+ # This method will make repeated API calls until all remaining results
86
+ # are retrieved. (Unlike `#each`, for example, which merely iterates
87
+ # over the results returned by a single API call.) Use with caution.
88
+ #
89
+ # @param [Integer] request_limit The upper limit of API requests to
90
+ # make to load all datasets. Default is no limit.
91
+ # @yield [dataset] The block for accessing each dataset.
92
+ # @yieldparam [Dataset] dataset The dataset object.
93
+ #
94
+ # @return [Enumerator]
95
+ #
96
+ # @example Iterating each result by passing a block:
97
+ # require "google/cloud/bigquery"
98
+ #
99
+ # bigquery = Google::Cloud::Bigquery.new
100
+ #
101
+ # bigquery.datasets.all do |dataset|
102
+ # puts dataset.name
103
+ # end
104
+ #
105
+ # @example Using the enumerator by not passing a block:
106
+ # require "google/cloud/bigquery"
107
+ #
108
+ # bigquery = Google::Cloud::Bigquery.new
109
+ #
110
+ # all_names = bigquery.datasets.all.map do |dataset|
111
+ # dataset.name
112
+ # end
113
+ #
114
+ # @example Limit the number of API calls made:
115
+ # require "google/cloud/bigquery"
116
+ #
117
+ # bigquery = Google::Cloud::Bigquery.new
118
+ #
119
+ # bigquery.datasets.all(request_limit: 10) do |dataset|
120
+ # puts dataset.name
121
+ # end
122
+ #
123
+ def all request_limit: nil
124
+ request_limit = request_limit.to_i if request_limit
125
+ return enum_for :all, request_limit: request_limit unless block_given?
126
+ results = self
127
+ loop do
128
+ results.each { |r| yield r }
129
+ if request_limit
130
+ request_limit -= 1
131
+ break if request_limit.negative?
132
+ end
133
+ break unless results.next?
134
+ results = results.next
135
+ end
136
+ end
137
+
138
+ ##
139
+ # @private New Dataset::List from a response object.
140
+ def self.from_gapi gapi_list, service, hidden = nil, filter = nil, max = nil
141
+ datasets = List.new(Array(gapi_list.datasets).map { |gapi_object| Dataset.from_gapi gapi_object, service })
142
+ datasets.instance_variable_set :@token, gapi_list.next_page_token
143
+ datasets.instance_variable_set :@etag, gapi_list.etag
144
+ datasets.instance_variable_set :@service, service
145
+ datasets.instance_variable_set :@hidden, hidden
146
+ datasets.instance_variable_set :@filter, filter
147
+ datasets.instance_variable_set :@max, max
148
+ datasets
149
+ end
150
+
151
+ protected
152
+
153
+ ##
154
+ # Raise an error unless an active service is available.
155
+ def ensure_service!
156
+ raise "Must have active connection" unless @service
157
+ end
158
+ end
159
+ end
160
+ end
161
+ end
162
+ end
@@ -0,0 +1,123 @@
1
+ # Copyright 2018 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ require "google/apis/bigquery_v2"
16
+
17
+ module Google
18
+ module Cloud
19
+ module Bigquery
20
+ ##
21
+ # # Encryption Configuration
22
+ #
23
+ # A builder for BigQuery table encryption configurations, passed to block
24
+ # arguments to {Dataset#create_table} and {Table#encryption}.
25
+ #
26
+ # @see https://cloud.google.com/bigquery/docs/customer-managed-encryption
27
+ # Protecting Data with Cloud KMS Keys
28
+ #
29
+ # @example
30
+ # require "google/cloud/bigquery"
31
+ #
32
+ # bigquery = Google::Cloud::Bigquery.new
33
+ # dataset = bigquery.dataset "my_dataset"
34
+ # key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
35
+ # encrypt_config = bigquery.encryption kms_key: key_name
36
+ # table = dataset.create_table "my_table" do |updater|
37
+ # updater.encryption = encrypt_config
38
+ # end
39
+ #
40
+ class EncryptionConfiguration
41
+ ##
42
+ # @private The Google API Client object.
43
+ attr_accessor :gapi
44
+
45
+ ##
46
+ # @private Create an empty EncryptionConfiguration object.
47
+ def initialize
48
+ @gapi = Google::Apis::BigqueryV2::EncryptionConfiguration.new
49
+ end
50
+
51
+ ##
52
+ # The Cloud KMS encryption key that will be used to protect the table.
53
+ # For example: `projects/a/locations/b/keyRings/c/cryptoKeys/d`
54
+ # The default value is `nil`, which means default encryption is used.
55
+ #
56
+ # @return [String]
57
+ #
58
+ # @example
59
+ # require "google/cloud/bigquery"
60
+ #
61
+ # config = Google::Cloud::Bigquery::EncryptionConfiguration.new
62
+ # key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
63
+ # config.kms_key = key_name
64
+ #
65
+ def kms_key
66
+ @gapi.kms_key_name
67
+ end
68
+
69
+ ##
70
+ # Set the Cloud KMS encryption key that will be used to protect the
71
+ # table. For example: `projects/a/locations/b/keyRings/c/cryptoKeys/d`
72
+ # The default value is `nil`, which means default encryption is used.
73
+ #
74
+ # @param [String] new_kms_key_name New Cloud KMS key name
75
+ #
76
+ # @example
77
+ # require "google/cloud/bigquery"
78
+ #
79
+ # config = Google::Cloud::Bigquery::EncryptionConfiguration.new
80
+ # key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
81
+ # config.kms_key = key_name
82
+ #
83
+ def kms_key= new_kms_key_name
84
+ frozen_check!
85
+ @gapi.kms_key_name = new_kms_key_name
86
+ end
87
+
88
+ # @private
89
+ def changed?
90
+ return false if frozen?
91
+ @original_json != @gapi.to_json
92
+ end
93
+
94
+ ##
95
+ # @private Google API Client object.
96
+ def to_gapi
97
+ @gapi
98
+ end
99
+
100
+ ##
101
+ # @private Google API Client object.
102
+ def self.from_gapi gapi
103
+ new_config = new
104
+ new_config.instance_variable_set :@gapi, gapi
105
+ new_config
106
+ end
107
+
108
+ # @private
109
+ def == other
110
+ return false unless other.is_a? EncryptionConfiguration
111
+ to_gapi.to_json == other.to_gapi.to_json
112
+ end
113
+
114
+ protected
115
+
116
+ def frozen_check!
117
+ return unless frozen?
118
+ raise ArgumentError, "Cannot modify a frozen encryption configuration"
119
+ end
120
+ end
121
+ end
122
+ end
123
+ end
@@ -0,0 +1,2432 @@
1
+ # Copyright 2017 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ require "google/apis/bigquery_v2"
17
+ require "base64"
18
+
19
+ module Google
20
+ module Cloud
21
+ module Bigquery
22
+ ##
23
+ # # External
24
+ #
25
+ # Creates a new {External::DataSource} (or subclass) object that
26
+ # represents the external data source that can be queried from directly,
27
+ # even though the data is not stored in BigQuery. Instead of loading or
28
+ # streaming the data, this object references the external data source.
29
+ #
30
+ # See {External::DataSource}, {External::CsvSource},
31
+ # {External::JsonSource}, {External::SheetsSource},
32
+ # {External::BigtableSource}
33
+ #
34
+ # @example
35
+ # require "google/cloud/bigquery"
36
+ #
37
+ # bigquery = Google::Cloud::Bigquery.new
38
+ #
39
+ # csv_url = "gs://bucket/path/to/data.csv"
40
+ # csv_table = bigquery.external csv_url do |csv|
41
+ # csv.autodetect = true
42
+ # csv.skip_leading_rows = 1
43
+ # end
44
+ #
45
+ # data = bigquery.query "SELECT * FROM my_ext_table",
46
+ # external: { my_ext_table: csv_table }
47
+ #
48
+ # # Iterate over the first page of results
49
+ # data.each do |row|
50
+ # puts row[:name]
51
+ # end
52
+ # # Retrieve the next page of results
53
+ # data = data.next if data.next?
54
+ #
55
+ module External
56
+ ##
57
+ # @private New External from URLs and format
58
+ def self.from_urls urls, format = nil
59
+ external_format = source_format_for urls, format
60
+ raise ArgumentError, "Unable to determine external table format" if external_format.nil?
61
+ external_class = table_class_for external_format
62
+ external_class.new.tap do |e|
63
+ e.gapi.source_uris = Array(urls)
64
+ e.gapi.source_format = external_format
65
+ end
66
+ end
67
+
68
+ ##
69
+ # @private Google API Client object.
70
+ def self.from_gapi gapi
71
+ external_format = source_format_for gapi.source_uris,
72
+ gapi.source_format
73
+ raise ArgumentError, "Unable to determine external table format" if external_format.nil?
74
+ external_class = table_class_for external_format
75
+ external_class.from_gapi gapi
76
+ end
77
+
78
+ ##
79
+ # @private Determine source_format from inputs
80
+ def self.source_format_for urls, format
81
+ val = {
82
+ "csv" => "CSV", "avro" => "AVRO",
83
+ "json" => "NEWLINE_DELIMITED_JSON",
84
+ "newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
85
+ "sheets" => "GOOGLE_SHEETS",
86
+ "google_sheets" => "GOOGLE_SHEETS",
87
+ "datastore" => "DATASTORE_BACKUP",
88
+ "backup" => "DATASTORE_BACKUP",
89
+ "datastore_backup" => "DATASTORE_BACKUP",
90
+ "bigtable" => "BIGTABLE"
91
+ }[format.to_s.downcase]
92
+ return val unless val.nil?
93
+ Array(urls).each do |url|
94
+ return "CSV" if url.end_with? ".csv"
95
+ return "NEWLINE_DELIMITED_JSON" if url.end_with? ".json"
96
+ return "AVRO" if url.end_with? ".avro"
97
+ return "DATASTORE_BACKUP" if url.end_with? ".backup_info"
98
+ return "GOOGLE_SHEETS" if url.start_with? "https://docs.google.com/spreadsheets/"
99
+ return "BIGTABLE" if url.start_with? "https://googleapis.com/bigtable/projects/"
100
+ end
101
+ nil
102
+ end
103
+
104
+ ##
105
+ # @private Determine table class from source_format
106
+ def self.table_class_for format
107
+ case format
108
+ when "CSV" then External::CsvSource
109
+ when "NEWLINE_DELIMITED_JSON" then External::JsonSource
110
+ when "GOOGLE_SHEETS" then External::SheetsSource
111
+ when "BIGTABLE" then External::BigtableSource
112
+ else
113
+ # AVRO and DATASTORE_BACKUP
114
+ External::DataSource
115
+ end
116
+ end
117
+
118
+ ##
119
+ # # DataSource
120
+ #
121
+ # External::DataSource and its subclasses represents an external data
122
+ # source that can be queried from directly, even though the data is not
123
+ # stored in BigQuery. Instead of loading or streaming the data, this
124
+ # object references the external data source.
125
+ #
126
+ # The AVRO and Datastore Backup formats use {External::DataSource}. See
127
+ # {External::CsvSource}, {External::JsonSource},
128
+ # {External::SheetsSource}, {External::BigtableSource} for the other
129
+ # formats.
130
+ #
131
+ # @example
132
+ # require "google/cloud/bigquery"
133
+ #
134
+ # bigquery = Google::Cloud::Bigquery.new
135
+ #
136
+ # avro_url = "gs://bucket/path/to/data.avro"
137
+ # avro_table = bigquery.external avro_url do |avro|
138
+ # avro.autodetect = true
139
+ # end
140
+ #
141
+ # data = bigquery.query "SELECT * FROM my_ext_table",
142
+ # external: { my_ext_table: avro_table }
143
+ #
144
+ # # Iterate over the first page of results
145
+ # data.each do |row|
146
+ # puts row[:name]
147
+ # end
148
+ # # Retrieve the next page of results
149
+ # data = data.next if data.next?
150
+ #
151
+ class DataSource
152
+ ##
153
+ # @private The Google API Client object.
154
+ attr_accessor :gapi
155
+
156
+ ##
157
+ # @private Create an empty Table object.
158
+ def initialize
159
+ @gapi = Google::Apis::BigqueryV2::ExternalDataConfiguration.new
160
+ end
161
+
162
+ ##
163
+ # The data format. For CSV files, specify "CSV". For Google sheets,
164
+ # specify "GOOGLE_SHEETS". For newline-delimited JSON, specify
165
+ # "NEWLINE_DELIMITED_JSON". For Avro files, specify "AVRO". For Google
166
+ # Cloud Datastore backups, specify "DATASTORE_BACKUP". [Beta] For
167
+ # Google Cloud Bigtable, specify "BIGTABLE".
168
+ #
169
+ # @return [String]
170
+ #
171
+ # @example
172
+ # require "google/cloud/bigquery"
173
+ #
174
+ # bigquery = Google::Cloud::Bigquery.new
175
+ #
176
+ # csv_url = "gs://bucket/path/to/data.csv"
177
+ # csv_table = bigquery.external csv_url
178
+ #
179
+ # csv_table.format #=> "CSV"
180
+ #
181
+ def format
182
+ @gapi.source_format
183
+ end
184
+
185
+ ##
186
+ # Whether the data format is "CSV".
187
+ #
188
+ # @return [Boolean]
189
+ #
190
+ # @example
191
+ # require "google/cloud/bigquery"
192
+ #
193
+ # bigquery = Google::Cloud::Bigquery.new
194
+ #
195
+ # csv_url = "gs://bucket/path/to/data.csv"
196
+ # csv_table = bigquery.external csv_url
197
+ #
198
+ # csv_table.format #=> "CSV"
199
+ # csv_table.csv? #=> true
200
+ #
201
+ def csv?
202
+ @gapi.source_format == "CSV"
203
+ end
204
+
205
+ ##
206
+ # Whether the data format is "NEWLINE_DELIMITED_JSON".
207
+ #
208
+ # @return [Boolean]
209
+ #
210
+ # @example
211
+ # require "google/cloud/bigquery"
212
+ #
213
+ # bigquery = Google::Cloud::Bigquery.new
214
+ #
215
+ # json_url = "gs://bucket/path/to/data.json"
216
+ # json_table = bigquery.external json_url
217
+ #
218
+ # json_table.format #=> "NEWLINE_DELIMITED_JSON"
219
+ # json_table.json? #=> true
220
+ #
221
+ def json?
222
+ @gapi.source_format == "NEWLINE_DELIMITED_JSON"
223
+ end
224
+
225
+ ##
226
+ # Whether the data format is "GOOGLE_SHEETS".
227
+ #
228
+ # @return [Boolean]
229
+ #
230
+ # @example
231
+ # require "google/cloud/bigquery"
232
+ #
233
+ # bigquery = Google::Cloud::Bigquery.new
234
+ #
235
+ # sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
236
+ # sheets_table = bigquery.external sheets_url
237
+ #
238
+ # sheets_table.format #=> "GOOGLE_SHEETS"
239
+ # sheets_table.sheets? #=> true
240
+ #
241
+ def sheets?
242
+ @gapi.source_format == "GOOGLE_SHEETS"
243
+ end
244
+
245
+ ##
246
+ # Whether the data format is "AVRO".
247
+ #
248
+ # @return [Boolean]
249
+ #
250
+ # @example
251
+ # require "google/cloud/bigquery"
252
+ #
253
+ # bigquery = Google::Cloud::Bigquery.new
254
+ #
255
+ # avro_url = "gs://bucket/path/to/data.avro"
256
+ # avro_table = bigquery.external avro_url
257
+ #
258
+ # avro_table.format #=> "AVRO"
259
+ # avro_table.avro? #=> true
260
+ #
261
+ def avro?
262
+ @gapi.source_format == "AVRO"
263
+ end
264
+
265
+ ##
266
+ # Whether the data format is "DATASTORE_BACKUP".
267
+ #
268
+ # @return [Boolean]
269
+ #
270
+ # @example
271
+ # require "google/cloud/bigquery"
272
+ #
273
+ # bigquery = Google::Cloud::Bigquery.new
274
+ #
275
+ # backup_url = "gs://bucket/path/to/data.backup_info"
276
+ # backup_table = bigquery.external backup_url
277
+ #
278
+ # backup_table.format #=> "DATASTORE_BACKUP"
279
+ # backup_table.backup? #=> true
280
+ #
281
+ def backup?
282
+ @gapi.source_format == "DATASTORE_BACKUP"
283
+ end
284
+
285
+ ##
286
+ # Whether the data format is "BIGTABLE".
287
+ #
288
+ # @return [Boolean]
289
+ #
290
+ # @example
291
+ # require "google/cloud/bigquery"
292
+ #
293
+ # bigquery = Google::Cloud::Bigquery.new
294
+ #
295
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
296
+ # bigtable_table = bigquery.external bigtable_url
297
+ #
298
+ # bigtable_table.format #=> "BIGTABLE"
299
+ # bigtable_table.bigtable? #=> true
300
+ #
301
+ def bigtable?
302
+ @gapi.source_format == "BIGTABLE"
303
+ end
304
+
305
+ ##
306
+ # The fully-qualified URIs that point to your data in Google Cloud.
307
+ # For Google Cloud Storage URIs: Each URI can contain one '*' wildcard
308
+ # character and it must come after the 'bucket' name. Size limits
309
+ # related to load jobs apply to external data sources. For Google
310
+ # Cloud Bigtable URIs: Exactly one URI can be specified and it has be
311
+ # a fully specified and valid HTTPS URL for a Google Cloud Bigtable
312
+ # table. For Google Cloud Datastore backups, exactly one URI can be
313
+ # specified, and it must end with '.backup_info'. Also, the '*'
314
+ # wildcard character is not allowed.
315
+ #
316
+ # @return [Array<String>]
317
+ #
318
+ # @example
319
+ # require "google/cloud/bigquery"
320
+ #
321
+ # bigquery = Google::Cloud::Bigquery.new
322
+ #
323
+ # csv_url = "gs://bucket/path/to/data.csv"
324
+ # csv_table = bigquery.external csv_url
325
+ #
326
+ # csv_table.urls #=> ["gs://bucket/path/to/data.csv"]
327
+ #
328
+ def urls
329
+ @gapi.source_uris
330
+ end
331
+
332
+ ##
333
+ # Indicates if the schema and format options are detected
334
+ # automatically.
335
+ #
336
+ # @return [Boolean]
337
+ #
338
+ # @example
339
+ # require "google/cloud/bigquery"
340
+ #
341
+ # bigquery = Google::Cloud::Bigquery.new
342
+ #
343
+ # csv_url = "gs://bucket/path/to/data.csv"
344
+ # csv_table = bigquery.external csv_url do |csv|
345
+ # csv.autodetect = true
346
+ # end
347
+ #
348
+ # csv_table.autodetect #=> true
349
+ #
350
+ def autodetect
351
+ @gapi.autodetect
352
+ end
353
+
354
+ ##
355
+ # Set whether to detect schema and format options automatically. Any
356
+ # option specified explicitly will be honored.
357
+ #
358
+ # @param [Boolean] new_autodetect New autodetect value
359
+ #
360
+ # @example
361
+ # require "google/cloud/bigquery"
362
+ #
363
+ # bigquery = Google::Cloud::Bigquery.new
364
+ #
365
+ # csv_url = "gs://bucket/path/to/data.csv"
366
+ # csv_table = bigquery.external csv_url do |csv|
367
+ # csv.autodetect = true
368
+ # end
369
+ #
370
+ # csv_table.autodetect #=> true
371
+ #
372
+ def autodetect= new_autodetect
373
+ frozen_check!
374
+ @gapi.autodetect = new_autodetect
375
+ end
376
+
377
+ ##
378
+ # The compression type of the data source. Possible values include
379
+ # `"GZIP"` and `nil`. The default value is `nil`. This setting is
380
+ # ignored for Google Cloud Bigtable, Google Cloud Datastore backups
381
+ # and Avro formats. Optional.
382
+ #
383
+ # @return [String]
384
+ #
385
+ # @example
386
+ # require "google/cloud/bigquery"
387
+ #
388
+ # bigquery = Google::Cloud::Bigquery.new
389
+ #
390
+ # csv_url = "gs://bucket/path/to/data.csv"
391
+ # csv_table = bigquery.external csv_url do |csv|
392
+ # csv.compression = "GZIP"
393
+ # end
394
+ #
395
+ # csv_table.compression #=> "GZIP"
396
+ def compression
397
+ @gapi.compression
398
+ end
399
+
400
+ ##
401
+ # Set the compression type of the data source. Possible values include
402
+ # `"GZIP"` and `nil`. The default value is `nil`. This setting is
403
+ # ignored for Google Cloud Bigtable, Google Cloud Datastore backups
404
+ # and Avro formats. Optional.
405
+ #
406
+ # @param [String] new_compression New compression value
407
+ #
408
+ # @example
409
+ # require "google/cloud/bigquery"
410
+ #
411
+ # bigquery = Google::Cloud::Bigquery.new
412
+ #
413
+ # csv_url = "gs://bucket/path/to/data.csv"
414
+ # csv_table = bigquery.external csv_url do |csv|
415
+ # csv.compression = "GZIP"
416
+ # end
417
+ #
418
+ # csv_table.compression #=> "GZIP"
419
+ #
420
+ def compression= new_compression
421
+ frozen_check!
422
+ @gapi.compression = new_compression
423
+ end
424
+
425
+ ##
426
+ # Indicates if BigQuery should allow extra values that are not
427
+ # represented in the table schema. If `true`, the extra values are
428
+ # ignored. If `false`, records with extra columns are treated as bad
429
+ # records, and if there are too many bad records, an invalid error is
430
+ # returned in the job result. The default value is `false`.
431
+ #
432
+ # BigQuery treats trailing columns as an extra in `CSV`, named values
433
+ # that don't match any column names in `JSON`. This setting is ignored
434
+ # for Google Cloud Bigtable, Google Cloud Datastore backups and Avro
435
+ # formats. Optional.
436
+ #
437
+ # @return [Boolean]
438
+ #
439
+ # @example
440
+ # require "google/cloud/bigquery"
441
+ #
442
+ # bigquery = Google::Cloud::Bigquery.new
443
+ #
444
+ # csv_url = "gs://bucket/path/to/data.csv"
445
+ # csv_table = bigquery.external csv_url do |csv|
446
+ # csv.ignore_unknown = true
447
+ # end
448
+ #
449
+ # csv_table.ignore_unknown #=> true
450
+ #
451
+ def ignore_unknown
452
+ @gapi.ignore_unknown_values
453
+ end
454
+
455
+ ##
456
+ # Set whether BigQuery should allow extra values that are not
457
+ # represented in the table schema. If `true`, the extra values are
458
+ # ignored. If `false`, records with extra columns are treated as bad
459
+ # records, and if there are too many bad records, an invalid error is
460
+ # returned in the job result. The default value is `false`.
461
+ #
462
+ # BigQuery treats trailing columns as an extra in `CSV`, named values
463
+ # that don't match any column names in `JSON`. This setting is ignored
464
+ # for Google Cloud Bigtable, Google Cloud Datastore backups and Avro
465
+ # formats. Optional.
466
+ #
467
+ # @param [Boolean] new_ignore_unknown New ignore_unknown value
468
+ #
469
+ # @example
470
+ # require "google/cloud/bigquery"
471
+ #
472
+ # bigquery = Google::Cloud::Bigquery.new
473
+ #
474
+ # csv_url = "gs://bucket/path/to/data.csv"
475
+ # csv_table = bigquery.external csv_url do |csv|
476
+ # csv.ignore_unknown = true
477
+ # end
478
+ #
479
+ # csv_table.ignore_unknown #=> true
480
+ #
481
+ def ignore_unknown= new_ignore_unknown
482
+ frozen_check!
483
+ @gapi.ignore_unknown_values = new_ignore_unknown
484
+ end
485
+
486
+ ##
487
+ # The maximum number of bad records that BigQuery can ignore when
488
+ # reading data. If the number of bad records exceeds this value, an
489
+ # invalid error is returned in the job result. The default value is 0,
490
+ # which requires that all records are valid. This setting is ignored
491
+ # for Google Cloud Bigtable, Google Cloud Datastore backups and Avro
492
+ # formats.
493
+ #
494
+ # @return [Integer]
495
+ #
496
+ # @example
497
+ # require "google/cloud/bigquery"
498
+ #
499
+ # bigquery = Google::Cloud::Bigquery.new
500
+ #
501
+ # csv_url = "gs://bucket/path/to/data.csv"
502
+ # csv_table = bigquery.external csv_url do |csv|
503
+ # csv.max_bad_records = 10
504
+ # end
505
+ #
506
+ # csv_table.max_bad_records #=> 10
507
+ #
508
+ def max_bad_records
509
+ @gapi.max_bad_records
510
+ end
511
+
512
+ ##
513
+ # Set the maximum number of bad records that BigQuery can ignore when
514
+ # reading data. If the number of bad records exceeds this value, an
515
+ # invalid error is returned in the job result. The default value is 0,
516
+ # which requires that all records are valid. This setting is ignored
517
+ # for Google Cloud Bigtable, Google Cloud Datastore backups and Avro
518
+ # formats.
519
+ #
520
+ # @param [Integer] new_max_bad_records New max_bad_records value
521
+ #
522
+ # @example
523
+ # require "google/cloud/bigquery"
524
+ #
525
+ # bigquery = Google::Cloud::Bigquery.new
526
+ #
527
+ # csv_url = "gs://bucket/path/to/data.csv"
528
+ # csv_table = bigquery.external csv_url do |csv|
529
+ # csv.max_bad_records = 10
530
+ # end
531
+ #
532
+ # csv_table.max_bad_records #=> 10
533
+ #
534
+ def max_bad_records= new_max_bad_records
535
+ frozen_check!
536
+ @gapi.max_bad_records = new_max_bad_records
537
+ end
538
+
539
+ ##
540
+ # @private Google API Client object.
541
+ def to_gapi
542
+ @gapi
543
+ end
544
+
545
+ ##
546
+ # @private Google API Client object.
547
+ def self.from_gapi gapi
548
+ new_table = new
549
+ new_table.instance_variable_set :@gapi, gapi
550
+ new_table
551
+ end
552
+
553
+ protected
554
+
555
+ def frozen_check!
556
+ return unless frozen?
557
+ raise ArgumentError, "Cannot modify external data source when frozen"
558
+ end
559
+ end
560
+
561
+ ##
562
+ # # CsvSource
563
+ #
564
+ # {External::CsvSource} is a subclass of {External::DataSource} and
565
+ # represents a CSV external data source that can be queried from
566
+ # directly, such as Google Cloud Storage or Google Drive, even though
567
+ # the data is not stored in BigQuery. Instead of loading or streaming
568
+ # the data, this object references the external data source.
569
+ #
570
+ # @example
571
+ # require "google/cloud/bigquery"
572
+ #
573
+ # bigquery = Google::Cloud::Bigquery.new
574
+ #
575
+ # csv_url = "gs://bucket/path/to/data.csv"
576
+ # csv_table = bigquery.external csv_url do |csv|
577
+ # csv.autodetect = true
578
+ # csv.skip_leading_rows = 1
579
+ # end
580
+ #
581
+ # data = bigquery.query "SELECT * FROM my_ext_table",
582
+ # external: { my_ext_table: csv_table }
583
+ #
584
+ # # Iterate over the first page of results
585
+ # data.each do |row|
586
+ # puts row[:name]
587
+ # end
588
+ # # Retrieve the next page of results
589
+ # data = data.next if data.next?
590
+ #
591
+ class CsvSource < External::DataSource
592
+ ##
593
+ # @private Create an empty CsvSource object.
594
+ def initialize
595
+ super
596
+ @gapi.csv_options = Google::Apis::BigqueryV2::CsvOptions.new
597
+ end
598
+
599
+ ##
600
+ # Indicates if BigQuery should accept rows that are missing trailing
601
+ # optional columns.
602
+ #
603
+ # @return [Boolean]
604
+ #
605
+ # @example
606
+ # require "google/cloud/bigquery"
607
+ #
608
+ # bigquery = Google::Cloud::Bigquery.new
609
+ #
610
+ # csv_url = "gs://bucket/path/to/data.csv"
611
+ # csv_table = bigquery.external csv_url do |csv|
612
+ # csv.jagged_rows = true
613
+ # end
614
+ #
615
+ # csv_table.jagged_rows #=> true
616
+ #
617
+ def jagged_rows
618
+ @gapi.csv_options.allow_jagged_rows
619
+ end
620
+
621
+ ##
622
+ # Set whether BigQuery should accept rows that are missing trailing
623
+ # optional columns.
624
+ #
625
+ # @param [Boolean] new_jagged_rows New jagged_rows value
626
+ #
627
+ # @example
628
+ # require "google/cloud/bigquery"
629
+ #
630
+ # bigquery = Google::Cloud::Bigquery.new
631
+ #
632
+ # csv_url = "gs://bucket/path/to/data.csv"
633
+ # csv_table = bigquery.external csv_url do |csv|
634
+ # csv.jagged_rows = true
635
+ # end
636
+ #
637
+ # csv_table.jagged_rows #=> true
638
+ #
639
+ def jagged_rows= new_jagged_rows
640
+ frozen_check!
641
+ @gapi.csv_options.allow_jagged_rows = new_jagged_rows
642
+ end
643
+
644
+ ##
645
+ # Indicates if BigQuery should allow quoted data sections that contain
646
+ # newline characters in a CSV file.
647
+ #
648
+ # @return [Boolean]
649
+ #
650
+ # @example
651
+ # require "google/cloud/bigquery"
652
+ #
653
+ # bigquery = Google::Cloud::Bigquery.new
654
+ #
655
+ # csv_url = "gs://bucket/path/to/data.csv"
656
+ # csv_table = bigquery.external csv_url do |csv|
657
+ # csv.quoted_newlines = true
658
+ # end
659
+ #
660
+ # csv_table.quoted_newlines #=> true
661
+ #
662
+ def quoted_newlines
663
+ @gapi.csv_options.allow_quoted_newlines
664
+ end
665
+
666
+ ##
667
+ # Set whether BigQuery should allow quoted data sections that contain
668
+ # newline characters in a CSV file.
669
+ #
670
+ # @param [Boolean] new_quoted_newlines New quoted_newlines value
671
+ #
672
+ # @example
673
+ # require "google/cloud/bigquery"
674
+ #
675
+ # bigquery = Google::Cloud::Bigquery.new
676
+ #
677
+ # csv_url = "gs://bucket/path/to/data.csv"
678
+ # csv_table = bigquery.external csv_url do |csv|
679
+ # csv.quoted_newlines = true
680
+ # end
681
+ #
682
+ # csv_table.quoted_newlines #=> true
683
+ #
684
+ def quoted_newlines= new_quoted_newlines
685
+ frozen_check!
686
+ @gapi.csv_options.allow_quoted_newlines = new_quoted_newlines
687
+ end
688
+
689
+ ##
690
+ # The character encoding of the data.
691
+ #
692
+ # @return [String]
693
+ #
694
+ # @example
695
+ # require "google/cloud/bigquery"
696
+ #
697
+ # bigquery = Google::Cloud::Bigquery.new
698
+ #
699
+ # csv_url = "gs://bucket/path/to/data.csv"
700
+ # csv_table = bigquery.external csv_url do |csv|
701
+ # csv.encoding = "UTF-8"
702
+ # end
703
+ #
704
+ # csv_table.encoding #=> "UTF-8"
705
+ #
706
+ def encoding
707
+ @gapi.csv_options.encoding
708
+ end
709
+
710
+ ##
711
+ # Set the character encoding of the data.
712
+ #
713
+ # @param [String] new_encoding New encoding value
714
+ #
715
+ # @example
716
+ # require "google/cloud/bigquery"
717
+ #
718
+ # bigquery = Google::Cloud::Bigquery.new
719
+ #
720
+ # csv_url = "gs://bucket/path/to/data.csv"
721
+ # csv_table = bigquery.external csv_url do |csv|
722
+ # csv.encoding = "UTF-8"
723
+ # end
724
+ #
725
+ # csv_table.encoding #=> "UTF-8"
726
+ #
727
+ def encoding= new_encoding
728
+ frozen_check!
729
+ @gapi.csv_options.encoding = new_encoding
730
+ end
731
+
732
+ ##
733
+ # Checks if the character encoding of the data is "UTF-8". This is the
734
+ # default.
735
+ #
736
+ # @return [Boolean]
737
+ #
738
+ # @example
739
+ # require "google/cloud/bigquery"
740
+ #
741
+ # bigquery = Google::Cloud::Bigquery.new
742
+ #
743
+ # csv_url = "gs://bucket/path/to/data.csv"
744
+ # csv_table = bigquery.external csv_url do |csv|
745
+ # csv.encoding = "UTF-8"
746
+ # end
747
+ #
748
+ # csv_table.encoding #=> "UTF-8"
749
+ # csv_table.utf8? #=> true
750
+ #
751
+ def utf8?
752
+ return true if encoding.nil?
753
+ encoding == "UTF-8"
754
+ end
755
+
756
+ ##
757
+ # Checks if the character encoding of the data is "ISO-8859-1".
758
+ #
759
+ # @return [Boolean]
760
+ #
761
+ # @example
762
+ # require "google/cloud/bigquery"
763
+ #
764
+ # bigquery = Google::Cloud::Bigquery.new
765
+ #
766
+ # csv_url = "gs://bucket/path/to/data.csv"
767
+ # csv_table = bigquery.external csv_url do |csv|
768
+ # csv.encoding = "ISO-8859-1"
769
+ # end
770
+ #
771
+ # csv_table.encoding #=> "ISO-8859-1"
772
+ # csv_table.iso8859_1? #=> true
773
+ #
774
+ def iso8859_1?
775
+ encoding == "ISO-8859-1"
776
+ end
777
+
778
+ ##
779
+ # The separator for fields in a CSV file.
780
+ #
781
+ # @return [String]
782
+ #
783
+ # @example
784
+ # require "google/cloud/bigquery"
785
+ #
786
+ # bigquery = Google::Cloud::Bigquery.new
787
+ #
788
+ # csv_url = "gs://bucket/path/to/data.csv"
789
+ # csv_table = bigquery.external csv_url do |csv|
790
+ # csv.delimiter = "|"
791
+ # end
792
+ #
793
+ # csv_table.delimiter #=> "|"
794
+ #
795
+ def delimiter
796
+ @gapi.csv_options.field_delimiter
797
+ end
798
+
799
+ ##
800
+ # Set the separator for fields in a CSV file.
801
+ #
802
+ # @param [String] new_delimiter New delimiter value
803
+ #
804
+ # @example
805
+ # require "google/cloud/bigquery"
806
+ #
807
+ # bigquery = Google::Cloud::Bigquery.new
808
+ #
809
+ # csv_url = "gs://bucket/path/to/data.csv"
810
+ # csv_table = bigquery.external csv_url do |csv|
811
+ # csv.delimiter = "|"
812
+ # end
813
+ #
814
+ # csv_table.delimiter #=> "|"
815
+ #
816
+ def delimiter= new_delimiter
817
+ frozen_check!
818
+ @gapi.csv_options.field_delimiter = new_delimiter
819
+ end
820
+
821
+ ##
822
+ # The value that is used to quote data sections in a CSV file.
823
+ #
824
+ # @return [String]
825
+ #
826
+ # @example
827
+ # require "google/cloud/bigquery"
828
+ #
829
+ # bigquery = Google::Cloud::Bigquery.new
830
+ #
831
+ # csv_url = "gs://bucket/path/to/data.csv"
832
+ # csv_table = bigquery.external csv_url do |csv|
833
+ # csv.quote = "'"
834
+ # end
835
+ #
836
+ # csv_table.quote #=> "'"
837
+ #
838
+ def quote
839
+ @gapi.csv_options.quote
840
+ end
841
+
842
+ ##
843
+ # Set the value that is used to quote data sections in a CSV file.
844
+ #
845
+ # @param [String] new_quote New quote value
846
+ #
847
+ # @example
848
+ # require "google/cloud/bigquery"
849
+ #
850
+ # bigquery = Google::Cloud::Bigquery.new
851
+ #
852
+ # csv_url = "gs://bucket/path/to/data.csv"
853
+ # csv_table = bigquery.external csv_url do |csv|
854
+ # csv.quote = "'"
855
+ # end
856
+ #
857
+ # csv_table.quote #=> "'"
858
+ #
859
+ def quote= new_quote
860
+ frozen_check!
861
+ @gapi.csv_options.quote = new_quote
862
+ end
863
+
864
+ ##
865
+ # The number of rows at the top of a CSV file that BigQuery will skip
866
+ # when reading the data.
867
+ #
868
+ # @return [Integer]
869
+ #
870
+ # @example
871
+ # require "google/cloud/bigquery"
872
+ #
873
+ # bigquery = Google::Cloud::Bigquery.new
874
+ #
875
+ # csv_url = "gs://bucket/path/to/data.csv"
876
+ # csv_table = bigquery.external csv_url do |csv|
877
+ # csv.skip_leading_rows = 1
878
+ # end
879
+ #
880
+ # csv_table.skip_leading_rows #=> 1
881
+ #
882
+ def skip_leading_rows
883
+ @gapi.csv_options.skip_leading_rows
884
+ end
885
+
886
+ ##
887
+ # Set the number of rows at the top of a CSV file that BigQuery will
888
+ # skip when reading the data.
889
+ #
890
+ # @param [Integer] row_count New skip_leading_rows value
891
+ #
892
+ # @example
893
+ # require "google/cloud/bigquery"
894
+ #
895
+ # bigquery = Google::Cloud::Bigquery.new
896
+ #
897
+ # csv_url = "gs://bucket/path/to/data.csv"
898
+ # csv_table = bigquery.external csv_url do |csv|
899
+ # csv.skip_leading_rows = 1
900
+ # end
901
+ #
902
+ # csv_table.skip_leading_rows #=> 1
903
+ #
904
+ def skip_leading_rows= row_count
905
+ frozen_check!
906
+ @gapi.csv_options.skip_leading_rows = row_count
907
+ end
908
+
909
+ ##
910
+ # The schema for the data.
911
+ #
912
+ # @param [Boolean] replace Whether to replace the existing schema with
913
+ # the new schema. If `true`, the fields will replace the existing
914
+ # schema. If `false`, the fields will be added to the existing
915
+ # schema. The default value is `false`.
916
+ # @yield [schema] a block for setting the schema
917
+ # @yieldparam [Schema] schema the object accepting the schema
918
+ #
919
+ # @return [Google::Cloud::Bigquery::Schema]
920
+ #
921
+ # @example
922
+ # require "google/cloud/bigquery"
923
+ #
924
+ # bigquery = Google::Cloud::Bigquery.new
925
+ #
926
+ # csv_url = "gs://bucket/path/to/data.csv"
927
+ # csv_table = bigquery.external csv_url do |csv|
928
+ # csv.schema do |schema|
929
+ # schema.string "name", mode: :required
930
+ # schema.string "email", mode: :required
931
+ # schema.integer "age", mode: :required
932
+ # schema.boolean "active", mode: :required
933
+ # end
934
+ # end
935
+ #
936
+ def schema replace: false
937
+ @schema ||= Schema.from_gapi @gapi.schema
938
+ if replace
939
+ frozen_check!
940
+ @schema = Schema.from_gapi
941
+ end
942
+ @schema.freeze if frozen?
943
+ yield @schema if block_given?
944
+ @schema
945
+ end
946
+
947
+ ##
948
+ # Set the schema for the data.
949
+ #
950
+ # @param [Schema] new_schema The schema object.
951
+ #
952
+ # @example
953
+ # require "google/cloud/bigquery"
954
+ #
955
+ # bigquery = Google::Cloud::Bigquery.new
956
+ #
957
+ # csv_shema = bigquery.schema do |schema|
958
+ # schema.string "name", mode: :required
959
+ # schema.string "email", mode: :required
960
+ # schema.integer "age", mode: :required
961
+ # schema.boolean "active", mode: :required
962
+ # end
963
+ #
964
+ # csv_url = "gs://bucket/path/to/data.csv"
965
+ # csv_table = bigquery.external csv_url
966
+ # csv_table.schema = csv_shema
967
+ #
968
+ def schema= new_schema
969
+ frozen_check!
970
+ @schema = new_schema
971
+ end
972
+
973
+ ##
974
+ # The fields of the schema.
975
+ #
976
+ # @return [Array<Schema::Field>] An array of field objects.
977
+ #
978
+ def fields
979
+ schema.fields
980
+ end
981
+
982
+ ##
983
+ # The names of the columns in the schema.
984
+ #
985
+ # @return [Array<Symbol>] An array of column names.
986
+ #
987
+ def headers
988
+ schema.headers
989
+ end
990
+
991
+ ##
992
+ # The types of the fields in the data in the schema, using the same
993
+ # format as the optional query parameter types.
994
+ #
995
+ # @return [Hash] A hash with field names as keys, and types as values.
996
+ #
997
+ def param_types
998
+ schema.param_types
999
+ end
1000
+
1001
+ ##
1002
+ # @private Google API Client object.
1003
+ def to_gapi
1004
+ @gapi.schema = @schema.to_gapi if @schema
1005
+ @gapi
1006
+ end
1007
+
1008
+ ##
1009
+ # @private Google API Client object.
1010
+ def self.from_gapi gapi
1011
+ new_table = super
1012
+ schema = Schema.from_gapi gapi.schema
1013
+ new_table.instance_variable_set :@schema, schema
1014
+ new_table
1015
+ end
1016
+ end
1017
+
1018
+ ##
1019
+ # # JsonSource
1020
+ #
1021
+ # {External::JsonSource} is a subclass of {External::DataSource} and
1022
+ # represents a JSON external data source that can be queried from
1023
+ # directly, such as Google Cloud Storage or Google Drive, even though
1024
+ # the data is not stored in BigQuery. Instead of loading or streaming
1025
+ # the data, this object references the external data source.
1026
+ #
1027
+ # @example
1028
+ # require "google/cloud/bigquery"
1029
+ #
1030
+ # bigquery = Google::Cloud::Bigquery.new
1031
+ #
1032
+ # require "google/cloud/bigquery"
1033
+ #
1034
+ # bigquery = Google::Cloud::Bigquery.new
1035
+ #
1036
+ # json_url = "gs://bucket/path/to/data.json"
1037
+ # json_table = bigquery.external json_url do |json|
1038
+ # json.schema do |schema|
1039
+ # schema.string "name", mode: :required
1040
+ # schema.string "email", mode: :required
1041
+ # schema.integer "age", mode: :required
1042
+ # schema.boolean "active", mode: :required
1043
+ # end
1044
+ # end
1045
+ #
1046
+ # data = bigquery.query "SELECT * FROM my_ext_table",
1047
+ # external: { my_ext_table: json_table }
1048
+ #
1049
+ # # Iterate over the first page of results
1050
+ # data.each do |row|
1051
+ # puts row[:name]
1052
+ # end
1053
+ # # Retrieve the next page of results
1054
+ # data = data.next if data.next?
1055
+ #
1056
+ class JsonSource < External::DataSource
1057
+ ##
1058
+ # The schema for the data.
1059
+ #
1060
+ # @param [Boolean] replace Whether to replace the existing schema with
1061
+ # the new schema. If `true`, the fields will replace the existing
1062
+ # schema. If `false`, the fields will be added to the existing
1063
+ # schema. The default value is `false`.
1064
+ # @yield [schema] a block for setting the schema
1065
+ # @yieldparam [Schema] schema the object accepting the schema
1066
+ #
1067
+ # @return [Google::Cloud::Bigquery::Schema]
1068
+ #
1069
+ # @example
1070
+ # require "google/cloud/bigquery"
1071
+ #
1072
+ # bigquery = Google::Cloud::Bigquery.new
1073
+ #
1074
+ # json_url = "gs://bucket/path/to/data.json"
1075
+ # json_table = bigquery.external json_url do |json|
1076
+ # json.schema do |schema|
1077
+ # schema.string "name", mode: :required
1078
+ # schema.string "email", mode: :required
1079
+ # schema.integer "age", mode: :required
1080
+ # schema.boolean "active", mode: :required
1081
+ # end
1082
+ # end
1083
+ #
1084
+ def schema replace: false
1085
+ @schema ||= Schema.from_gapi @gapi.schema
1086
+ if replace
1087
+ frozen_check!
1088
+ @schema = Schema.from_gapi
1089
+ end
1090
+ @schema.freeze if frozen?
1091
+ yield @schema if block_given?
1092
+ @schema
1093
+ end
1094
+
1095
+ ##
1096
+ # Set the schema for the data.
1097
+ #
1098
+ # @param [Schema] new_schema The schema object.
1099
+ #
1100
+ # @example
1101
+ # require "google/cloud/bigquery"
1102
+ #
1103
+ # bigquery = Google::Cloud::Bigquery.new
1104
+ #
1105
+ # json_shema = bigquery.schema do |schema|
1106
+ # schema.string "name", mode: :required
1107
+ # schema.string "email", mode: :required
1108
+ # schema.integer "age", mode: :required
1109
+ # schema.boolean "active", mode: :required
1110
+ # end
1111
+ #
1112
+ # json_url = "gs://bucket/path/to/data.json"
1113
+ # json_table = bigquery.external json_url
1114
+ # json_table.schema = json_shema
1115
+ #
1116
+ def schema= new_schema
1117
+ frozen_check!
1118
+ @schema = new_schema
1119
+ end
1120
+
1121
+ ##
1122
+ # The fields of the schema.
1123
+ #
1124
+ # @return [Array<Schema::Field>] An array of field objects.
1125
+ #
1126
+ def fields
1127
+ schema.fields
1128
+ end
1129
+
1130
+ ##
1131
+ # The names of the columns in the schema.
1132
+ #
1133
+ # @return [Array<Symbol>] An array of column names.
1134
+ #
1135
+ def headers
1136
+ schema.headers
1137
+ end
1138
+
1139
+ ##
1140
+ # The types of the fields in the data in the schema, using the same
1141
+ # format as the optional query parameter types.
1142
+ #
1143
+ # @return [Hash] A hash with field names as keys, and types as values.
1144
+ #
1145
+ def param_types
1146
+ schema.param_types
1147
+ end
1148
+
1149
+ ##
1150
+ # @private Google API Client object.
1151
+ def to_gapi
1152
+ @gapi.schema = @schema.to_gapi if @schema
1153
+ @gapi
1154
+ end
1155
+
1156
+ ##
1157
+ # @private Google API Client object.
1158
+ def self.from_gapi gapi
1159
+ new_table = super
1160
+ schema = Schema.from_gapi gapi.schema
1161
+ new_table.instance_variable_set :@schema, schema
1162
+ new_table
1163
+ end
1164
+ end
1165
+
1166
+ ##
1167
+ # # SheetsSource
1168
+ #
1169
+ # {External::SheetsSource} is a subclass of {External::DataSource} and
1170
+ # represents a Google Sheets external data source that can be queried
1171
+ # from directly, even though the data is not stored in BigQuery. Instead
1172
+ # of loading or streaming the data, this object references the external
1173
+ # data source.
1174
+ #
1175
+ # @example
1176
+ # require "google/cloud/bigquery"
1177
+ #
1178
+ # bigquery = Google::Cloud::Bigquery.new
1179
+ #
1180
+ # sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
1181
+ # sheets_table = bigquery.external sheets_url do |sheets|
1182
+ # sheets.skip_leading_rows = 1
1183
+ # end
1184
+ #
1185
+ # data = bigquery.query "SELECT * FROM my_ext_table",
1186
+ # external: { my_ext_table: sheets_table }
1187
+ #
1188
+ # # Iterate over the first page of results
1189
+ # data.each do |row|
1190
+ # puts row[:name]
1191
+ # end
1192
+ # # Retrieve the next page of results
1193
+ # data = data.next if data.next?
1194
+ #
1195
+ class SheetsSource < External::DataSource
1196
+ ##
1197
+ # @private Create an empty SheetsSource object.
1198
+ def initialize
1199
+ super
1200
+ @gapi.google_sheets_options = Google::Apis::BigqueryV2::GoogleSheetsOptions.new
1201
+ end
1202
+
1203
+ ##
1204
+ # The number of rows at the top of a sheet that BigQuery will skip
1205
+ # when reading the data. The default value is `0`.
1206
+ #
1207
+ # This property is useful if you have header rows that should be
1208
+ # skipped. When `autodetect` is on, behavior is the following:
1209
+ #
1210
+ # * `nil` - Autodetect tries to detect headers in the first row. If
1211
+ # they are not detected, the row is read as data. Otherwise data is
1212
+ # read starting from the second row.
1213
+ # * `0` - Instructs autodetect that there are no headers and data
1214
+ # should be read starting from the first row.
1215
+ # * `N > 0` - Autodetect skips `N-1` rows and tries to detect headers
1216
+ # in row `N`. If headers are not detected, row `N` is just skipped.
1217
+ # Otherwise row `N` is used to extract column names for the detected
1218
+ # schema.
1219
+ #
1220
+ # @return [Integer]
1221
+ #
1222
+ # @example
1223
+ # require "google/cloud/bigquery"
1224
+ #
1225
+ # bigquery = Google::Cloud::Bigquery.new
1226
+ #
1227
+ # sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
1228
+ # sheets_table = bigquery.external sheets_url do |sheets|
1229
+ # sheets.skip_leading_rows = 1
1230
+ # end
1231
+ #
1232
+ # sheets_table.skip_leading_rows #=> 1
1233
+ #
1234
+ def skip_leading_rows
1235
+ @gapi.google_sheets_options.skip_leading_rows
1236
+ end
1237
+
1238
+ ##
1239
+ # Set the number of rows at the top of a sheet that BigQuery will skip
1240
+ # when reading the data.
1241
+ #
1242
+ # @param [Integer] row_count New skip_leading_rows value
1243
+ #
1244
+ # @example
1245
+ # require "google/cloud/bigquery"
1246
+ #
1247
+ # bigquery = Google::Cloud::Bigquery.new
1248
+ #
1249
+ # sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
1250
+ # sheets_table = bigquery.external sheets_url do |sheets|
1251
+ # sheets.skip_leading_rows = 1
1252
+ # end
1253
+ #
1254
+ # sheets_table.skip_leading_rows #=> 1
1255
+ #
1256
+ def skip_leading_rows= row_count
1257
+ frozen_check!
1258
+ @gapi.google_sheets_options.skip_leading_rows = row_count
1259
+ end
1260
+
1261
+ ##
1262
+ # Range of a sheet to query from. Only used when non-empty. Typical
1263
+ # format: `{sheet_name}!{top_left_cell_id}:{bottom_right_cell_id}`.
1264
+ #
1265
+ # @return [String] Range of a sheet to query from.
1266
+ #
1267
+ # @example
1268
+ # require "google/cloud/bigquery"
1269
+ #
1270
+ # bigquery = Google::Cloud::Bigquery.new
1271
+ #
1272
+ # sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
1273
+ # sheets_table = bigquery.external sheets_url do |sheets|
1274
+ # sheets.range = "sheet1!A1:B20"
1275
+ # end
1276
+ #
1277
+ # sheets_table.range #=> "sheet1!A1:B20"
1278
+ #
1279
+ def range
1280
+ @gapi.google_sheets_options.range
1281
+ end
1282
+
1283
+ ##
1284
+ # Set the range of a sheet to query from. Only used when non-empty.
1285
+ # Typical format:
1286
+ # `{sheet_name}!{top_left_cell_id}:{bottom_right_cell_id}`.
1287
+ #
1288
+ # @param [String] new_range New range of a sheet to query from.
1289
+ #
1290
+ # @example
1291
+ # require "google/cloud/bigquery"
1292
+ #
1293
+ # bigquery = Google::Cloud::Bigquery.new
1294
+ #
1295
+ # sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
1296
+ # sheets_table = bigquery.external sheets_url do |sheets|
1297
+ # sheets.range = "sheet1!A1:B20"
1298
+ # end
1299
+ #
1300
+ # sheets_table.range #=> "sheet1!A1:B20"
1301
+ #
1302
+ def range= new_range
1303
+ frozen_check!
1304
+ @gapi.google_sheets_options.range = new_range
1305
+ end
1306
+ end
1307
+
1308
+ ##
1309
+ # # BigtableSource
1310
+ #
1311
+ # {External::BigtableSource} is a subclass of {External::DataSource} and
1312
+ # represents a Bigtable external data source that can be queried from
1313
+ # directly, even though the data is not stored in BigQuery. Instead of
1314
+ # loading or streaming the data, this object references the external
1315
+ # data source.
1316
+ #
1317
+ # @example
1318
+ # require "google/cloud/bigquery"
1319
+ #
1320
+ # bigquery = Google::Cloud::Bigquery.new
1321
+ #
1322
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1323
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1324
+ # bt.rowkey_as_string = true
1325
+ # bt.add_family "user" do |u|
1326
+ # u.add_string "name"
1327
+ # u.add_string "email"
1328
+ # u.add_integer "age"
1329
+ # u.add_boolean "active"
1330
+ # end
1331
+ # end
1332
+ #
1333
+ # data = bigquery.query "SELECT * FROM my_ext_table",
1334
+ # external: { my_ext_table: bigtable_table }
1335
+ #
1336
+ # # Iterate over the first page of results
1337
+ # data.each do |row|
1338
+ # puts row[:name]
1339
+ # end
1340
+ # # Retrieve the next page of results
1341
+ # data = data.next if data.next?
1342
+ #
1343
+ class BigtableSource < External::DataSource
1344
+ ##
1345
+ # @private Create an empty BigtableSource object.
1346
+ def initialize
1347
+ super
1348
+ @gapi.bigtable_options = Google::Apis::BigqueryV2::BigtableOptions.new
1349
+ @families = []
1350
+ end
1351
+
1352
+ ##
1353
+ # List of column families to expose in the table schema along with
1354
+ # their types. This list restricts the column families that can be
1355
+ # referenced in queries and specifies their value types. You can use
1356
+ # this list to do type conversions - see
1357
+ # {BigtableSource::ColumnFamily#type} for more details. If you leave
1358
+ # this list empty, all column families are present in the table schema
1359
+ # and their values are read as `BYTES`. During a query only the column
1360
+ # families referenced in that query are read from Bigtable.
1361
+ #
1362
+ # @return [Array<BigtableSource::ColumnFamily>]
1363
+ #
1364
+ # @example
1365
+ # require "google/cloud/bigquery"
1366
+ #
1367
+ # bigquery = Google::Cloud::Bigquery.new
1368
+ #
1369
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1370
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1371
+ # bt.rowkey_as_string = true
1372
+ # bt.add_family "user" do |u|
1373
+ # u.add_string "name"
1374
+ # u.add_string "email"
1375
+ # u.add_integer "age"
1376
+ # u.add_boolean "active"
1377
+ # end
1378
+ # end
1379
+ #
1380
+ # bigtable_table.families.count #=> 1
1381
+ #
1382
+ def families
1383
+ @families
1384
+ end
1385
+
1386
+ ##
1387
+ # Add a column family to expose in the table schema along with its
1388
+ # types. Columns belonging to the column family may also be exposed.
1389
+ #
1390
+ # @param [String] family_id Identifier of the column family. See
1391
+ # {BigtableSource::ColumnFamily#family_id}.
1392
+ # @param [String] encoding The encoding of the values when the type is
1393
+ # not `STRING`. See {BigtableSource::ColumnFamily#encoding}.
1394
+ # @param [Boolean] latest Whether only the latest version of value are
1395
+ # exposed for all columns in this column family. See
1396
+ # {BigtableSource::ColumnFamily#latest}.
1397
+ # @param [String] type The type to convert the value in cells of this
1398
+ # column. See {BigtableSource::ColumnFamily#type}.
1399
+ #
1400
+ # @yield [family] a block for setting the family
1401
+ # @yieldparam [BigtableSource::ColumnFamily] family the family object
1402
+ #
1403
+ # @return [BigtableSource::ColumnFamily]
1404
+ #
1405
+ # @example
1406
+ # require "google/cloud/bigquery"
1407
+ #
1408
+ # bigquery = Google::Cloud::Bigquery.new
1409
+ #
1410
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1411
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1412
+ # bt.rowkey_as_string = true
1413
+ # bt.add_family "user" do |u|
1414
+ # u.add_string "name"
1415
+ # u.add_string "email"
1416
+ # u.add_integer "age"
1417
+ # u.add_boolean "active"
1418
+ # end
1419
+ # end
1420
+ #
1421
+ def add_family family_id, encoding: nil, latest: nil, type: nil
1422
+ frozen_check!
1423
+ fam = BigtableSource::ColumnFamily.new
1424
+ fam.family_id = family_id
1425
+ fam.encoding = encoding if encoding
1426
+ fam.latest = latest if latest
1427
+ fam.type = type if type
1428
+ yield fam if block_given?
1429
+ @families << fam
1430
+ fam
1431
+ end
1432
+
1433
+ ##
1434
+ # Whether the rowkey column families will be read and converted to
1435
+ # string. Otherwise they are read with `BYTES` type values and users
1436
+ # need to manually cast them with `CAST` if necessary. The default
1437
+ # value is `false`.
1438
+ #
1439
+ # @return [Boolean]
1440
+ #
1441
+ # @example
1442
+ # require "google/cloud/bigquery"
1443
+ #
1444
+ # bigquery = Google::Cloud::Bigquery.new
1445
+ #
1446
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1447
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1448
+ # bt.rowkey_as_string = true
1449
+ # end
1450
+ #
1451
+ # bigtable_table.rowkey_as_string #=> true
1452
+ #
1453
+ def rowkey_as_string
1454
+ @gapi.bigtable_options.read_rowkey_as_string
1455
+ end
1456
+
1457
+ ##
1458
+ # Set the number of rows at the top of a sheet that BigQuery will skip
1459
+ # when reading the data.
1460
+ #
1461
+ # @param [Boolean] row_rowkey New rowkey_as_string value
1462
+ #
1463
+ # @example
1464
+ # require "google/cloud/bigquery"
1465
+ #
1466
+ # bigquery = Google::Cloud::Bigquery.new
1467
+ #
1468
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1469
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1470
+ # bt.rowkey_as_string = true
1471
+ # end
1472
+ #
1473
+ # bigtable_table.rowkey_as_string #=> true
1474
+ #
1475
+ def rowkey_as_string= row_rowkey
1476
+ frozen_check!
1477
+ @gapi.bigtable_options.read_rowkey_as_string = row_rowkey
1478
+ end
1479
+
1480
+ ##
1481
+ # @private Google API Client object.
1482
+ def to_gapi
1483
+ @gapi.bigtable_options.column_families = @families.map(&:to_gapi)
1484
+ @gapi
1485
+ end
1486
+
1487
+ ##
1488
+ # @private Google API Client object.
1489
+ def self.from_gapi gapi
1490
+ new_table = super
1491
+ families = Array gapi.bigtable_options.column_families
1492
+ families = families.map { |fam_gapi| BigtableSource::ColumnFamily.from_gapi fam_gapi }
1493
+ new_table.instance_variable_set :@families, families
1494
+ new_table
1495
+ end
1496
+
1497
+ ##
1498
+ # @private
1499
+ def freeze
1500
+ @families.map(&:freeze!)
1501
+ @families.freeze!
1502
+ super
1503
+ end
1504
+
1505
+ protected
1506
+
1507
+ def frozen_check!
1508
+ return unless frozen?
1509
+ raise ArgumentError, "Cannot modify external data source when frozen"
1510
+ end
1511
+
1512
+ ##
1513
+ # # BigtableSource::ColumnFamily
1514
+ #
1515
+ # A Bigtable column family used to expose in the table schema along
1516
+ # with its types and columns.
1517
+ #
1518
+ # @example
1519
+ # require "google/cloud/bigquery"
1520
+ #
1521
+ # bigquery = Google::Cloud::Bigquery.new
1522
+ #
1523
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1524
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1525
+ # bt.rowkey_as_string = true
1526
+ # bt.add_family "user" do |u|
1527
+ # u.add_string "name"
1528
+ # u.add_string "email"
1529
+ # u.add_integer "age"
1530
+ # u.add_boolean "active"
1531
+ # end
1532
+ # end
1533
+ #
1534
+ # data = bigquery.query "SELECT * FROM my_ext_table",
1535
+ # external: { my_ext_table: bigtable_table }
1536
+ #
1537
+ # # Iterate over the first page of results
1538
+ # data.each do |row|
1539
+ # puts row[:name]
1540
+ # end
1541
+ # # Retrieve the next page of results
1542
+ # data = data.next if data.next?
1543
+ #
1544
+ class ColumnFamily
1545
+ ##
1546
+ # @private Create an empty BigtableSource::ColumnFamily object.
1547
+ def initialize
1548
+ @gapi = Google::Apis::BigqueryV2::BigtableColumnFamily.new
1549
+ @columns = []
1550
+ end
1551
+
1552
+ ##
1553
+ # The encoding of the values when the type is not `STRING`.
1554
+ #
1555
+ # @return [String]
1556
+ #
1557
+ # @example
1558
+ # require "google/cloud/bigquery"
1559
+ #
1560
+ # bigquery = Google::Cloud::Bigquery.new
1561
+ #
1562
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1563
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1564
+ # bt.add_family "user" do |u|
1565
+ # u.encoding = "UTF-8"
1566
+ # end
1567
+ # end
1568
+ #
1569
+ # bigtable_table.families[0].encoding #=> "UTF-8"
1570
+ #
1571
+ def encoding
1572
+ @gapi.encoding
1573
+ end
1574
+
1575
+ ##
1576
+ # Set the encoding of the values when the type is not `STRING`.
1577
+ # Acceptable encoding values are:
1578
+ #
1579
+ # * `TEXT` - indicates values are alphanumeric text strings.
1580
+ # * `BINARY` - indicates values are encoded using HBase
1581
+ # `Bytes.toBytes` family of functions. This can be overridden on a
1582
+ # column.
1583
+ #
1584
+ # @param [String] new_encoding New encoding value
1585
+ #
1586
+ # @example
1587
+ # require "google/cloud/bigquery"
1588
+ #
1589
+ # bigquery = Google::Cloud::Bigquery.new
1590
+ #
1591
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1592
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1593
+ # bt.add_family "user" do |u|
1594
+ # u.encoding = "UTF-8"
1595
+ # end
1596
+ # end
1597
+ #
1598
+ # bigtable_table.families[0].encoding #=> "UTF-8"
1599
+ #
1600
+ def encoding= new_encoding
1601
+ frozen_check!
1602
+ @gapi.encoding = new_encoding
1603
+ end
1604
+
1605
+ ##
1606
+ # Identifier of the column family.
1607
+ #
1608
+ # @return [String]
1609
+ #
1610
+ # @example
1611
+ # require "google/cloud/bigquery"
1612
+ #
1613
+ # bigquery = Google::Cloud::Bigquery.new
1614
+ #
1615
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1616
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1617
+ # bt.add_family "user"
1618
+ # end
1619
+ #
1620
+ # bigtable_table.families[0].family_id #=> "user"
1621
+ #
1622
+ def family_id
1623
+ @gapi.family_id
1624
+ end
1625
+
1626
+ ##
1627
+ # Set the identifier of the column family.
1628
+ #
1629
+ # @param [String] new_family_id New family_id value
1630
+ #
1631
+ # @example
1632
+ # require "google/cloud/bigquery"
1633
+ #
1634
+ # bigquery = Google::Cloud::Bigquery.new
1635
+ #
1636
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1637
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1638
+ # bt.add_family "user"
1639
+ # end
1640
+ #
1641
+ # bigtable_table.families[0].family_id #=> "user"
1642
+ # bigtable_table.families[0].family_id = "User"
1643
+ # bigtable_table.families[0].family_id #=> "User"
1644
+ #
1645
+ def family_id= new_family_id
1646
+ frozen_check!
1647
+ @gapi.family_id = new_family_id
1648
+ end
1649
+
1650
+ ##
1651
+ # Whether only the latest version of value are exposed for all
1652
+ # columns in this column family.
1653
+ #
1654
+ # @return [Boolean]
1655
+ #
1656
+ # @example
1657
+ # require "google/cloud/bigquery"
1658
+ #
1659
+ # bigquery = Google::Cloud::Bigquery.new
1660
+ #
1661
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1662
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1663
+ # bt.add_family "user" do |u|
1664
+ # u.latest = true
1665
+ # end
1666
+ # end
1667
+ #
1668
+ # bigtable_table.families[0].latest #=> true
1669
+ #
1670
+ def latest
1671
+ @gapi.only_read_latest
1672
+ end
1673
+
1674
+ ##
1675
+ # Set whether only the latest version of value are exposed for all
1676
+ # columns in this column family.
1677
+ #
1678
+ # @param [Boolean] new_latest New latest value
1679
+ #
1680
+ # @example
1681
+ # require "google/cloud/bigquery"
1682
+ #
1683
+ # bigquery = Google::Cloud::Bigquery.new
1684
+ #
1685
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1686
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1687
+ # bt.add_family "user" do |u|
1688
+ # u.latest = true
1689
+ # end
1690
+ # end
1691
+ #
1692
+ # bigtable_table.families[0].latest #=> true
1693
+ #
1694
+ def latest= new_latest
1695
+ frozen_check!
1696
+ @gapi.only_read_latest = new_latest
1697
+ end
1698
+
1699
+ ##
1700
+ # The type to convert the value in cells of this column family. The
1701
+ # values are expected to be encoded using HBase `Bytes.toBytes`
1702
+ # function when using the `BINARY` encoding value. The following
1703
+ # BigQuery types are allowed:
1704
+ #
1705
+ # * `BYTES`
1706
+ # * `STRING`
1707
+ # * `INTEGER`
1708
+ # * `FLOAT`
1709
+ # * `BOOLEAN`
1710
+ #
1711
+ # Default type is `BYTES`. This can be overridden on a column.
1712
+ #
1713
+ # @return [String]
1714
+ #
1715
+ # @example
1716
+ # require "google/cloud/bigquery"
1717
+ #
1718
+ # bigquery = Google::Cloud::Bigquery.new
1719
+ #
1720
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1721
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1722
+ # bt.add_family "user" do |u|
1723
+ # u.type = "STRING"
1724
+ # end
1725
+ # end
1726
+ #
1727
+ # bigtable_table.families[0].type #=> "STRING"
1728
+ #
1729
+ def type
1730
+ @gapi.type
1731
+ end
1732
+
1733
+ ##
1734
+ # Set the type to convert the value in cells of this column family.
1735
+ # The values are expected to be encoded using HBase `Bytes.toBytes`
1736
+ # function when using the `BINARY` encoding value. The following
1737
+ # BigQuery types are allowed:
1738
+ #
1739
+ # * `BYTES`
1740
+ # * `STRING`
1741
+ # * `INTEGER`
1742
+ # * `FLOAT`
1743
+ # * `BOOLEAN`
1744
+ #
1745
+ # Default type is `BYTES`. This can be overridden on a column.
1746
+ #
1747
+ # @param [String] new_type New type value
1748
+ #
1749
+ # @example
1750
+ # require "google/cloud/bigquery"
1751
+ #
1752
+ # bigquery = Google::Cloud::Bigquery.new
1753
+ #
1754
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1755
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1756
+ # bt.add_family "user" do |u|
1757
+ # u.type = "STRING"
1758
+ # end
1759
+ # end
1760
+ #
1761
+ # bigtable_table.families[0].type #=> "STRING"
1762
+ #
1763
+ def type= new_type
1764
+ frozen_check!
1765
+ @gapi.type = new_type
1766
+ end
1767
+
1768
+ ##
1769
+ # Lists of columns that should be exposed as individual fields.
1770
+ #
1771
+ # @return [Array<BigtableSource::Column>]
1772
+ #
1773
+ # @example
1774
+ # require "google/cloud/bigquery"
1775
+ #
1776
+ # bigquery = Google::Cloud::Bigquery.new
1777
+ #
1778
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1779
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1780
+ # bt.rowkey_as_string = true
1781
+ # bt.add_family "user" do |u|
1782
+ # u.add_string "name"
1783
+ # u.add_string "email"
1784
+ # u.add_integer "age"
1785
+ # u.add_boolean "active"
1786
+ # end
1787
+ # end
1788
+ #
1789
+ # bigtable_table.families[0].columns.count #=> 4
1790
+ #
1791
+ def columns
1792
+ @columns
1793
+ end
1794
+
1795
+ ##
1796
+ # Add a column to the column family to expose in the table schema
1797
+ # along with its types.
1798
+ #
1799
+ # @param [String] qualifier Qualifier of the column. See
1800
+ # {BigtableSource::Column#qualifier}.
1801
+ # @param [String] as A valid identifier to be used as the column
1802
+ # field name if the qualifier is not a valid BigQuery field
1803
+ # identifier (i.e. does not match `[a-zA-Z][a-zA-Z0-9_]*`). See
1804
+ # {BigtableSource::Column#field_name}.
1805
+ # @param [String] type The type to convert the value in cells of
1806
+ # this column. See {BigtableSource::Column#type}. The following
1807
+ # BigQuery types are allowed:
1808
+ #
1809
+ # * `BYTES`
1810
+ # * `STRING`
1811
+ # * `INTEGER`
1812
+ # * `FLOAT`
1813
+ # * `BOOLEAN`
1814
+ #
1815
+ # @yield [column] a block for setting the column
1816
+ # @yieldparam [BigtableSource::Column] column the column object
1817
+ #
1818
+ # @return [Array<BigtableSource::Column>]
1819
+ #
1820
+ # @example
1821
+ # require "google/cloud/bigquery"
1822
+ #
1823
+ # bigquery = Google::Cloud::Bigquery.new
1824
+ #
1825
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1826
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1827
+ # bt.rowkey_as_string = true
1828
+ # bt.add_family "user" do |u|
1829
+ # u.add_column "name", type: "STRING"
1830
+ # end
1831
+ # end
1832
+ #
1833
+ def add_column qualifier, as: nil, type: nil
1834
+ frozen_check!
1835
+ col = BigtableSource::Column.new
1836
+ col.qualifier = qualifier
1837
+ col.field_name = as if as
1838
+ col.type = type if type
1839
+ yield col if block_given?
1840
+ @columns << col
1841
+ col
1842
+ end
1843
+
1844
+ ##
1845
+ # Add a column to the column family to expose in the table schema
1846
+ # that is specified as the `BYTES` type.
1847
+ #
1848
+ # @param [String] qualifier Qualifier of the column. See
1849
+ # {BigtableSource::Column#qualifier}.
1850
+ # @param [String] as A valid identifier to be used as the column
1851
+ # field name if the qualifier is not a valid BigQuery field
1852
+ # identifier (i.e. does not match `[a-zA-Z][a-zA-Z0-9_]*`). See
1853
+ # {BigtableSource::Column#field_name}.
1854
+ #
1855
+ # @yield [column] a block for setting the column
1856
+ # @yieldparam [BigtableSource::Column] column the column object
1857
+ #
1858
+ # @return [Array<BigtableSource::Column>]
1859
+ #
1860
+ # @example
1861
+ # require "google/cloud/bigquery"
1862
+ #
1863
+ # bigquery = Google::Cloud::Bigquery.new
1864
+ #
1865
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1866
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1867
+ # bt.rowkey_as_string = true
1868
+ # bt.add_family "user" do |u|
1869
+ # u.add_bytes "avatar"
1870
+ # end
1871
+ # end
1872
+ #
1873
+ def add_bytes qualifier, as: nil
1874
+ col = add_column qualifier, as: as, type: "BYTES"
1875
+ yield col if block_given?
1876
+ col
1877
+ end
1878
+
1879
+ ##
1880
+ # Add a column to the column family to expose in the table schema
1881
+ # that is specified as the `STRING` type.
1882
+ #
1883
+ # @param [String] qualifier Qualifier of the column. See
1884
+ # {BigtableSource::Column#qualifier}.
1885
+ # @param [String] as A valid identifier to be used as the column
1886
+ # field name if the qualifier is not a valid BigQuery field
1887
+ # identifier (i.e. does not match `[a-zA-Z][a-zA-Z0-9_]*`). See
1888
+ # {BigtableSource::Column#field_name}.
1889
+ #
1890
+ # @yield [column] a block for setting the column
1891
+ # @yieldparam [BigtableSource::Column] column the column object
1892
+ #
1893
+ # @return [Array<BigtableSource::Column>]
1894
+ #
1895
+ # @example
1896
+ # require "google/cloud/bigquery"
1897
+ #
1898
+ # bigquery = Google::Cloud::Bigquery.new
1899
+ #
1900
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1901
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1902
+ # bt.rowkey_as_string = true
1903
+ # bt.add_family "user" do |u|
1904
+ # u.add_string "name"
1905
+ # end
1906
+ # end
1907
+ #
1908
+ def add_string qualifier, as: nil
1909
+ col = add_column qualifier, as: as, type: "STRING"
1910
+ yield col if block_given?
1911
+ col
1912
+ end
1913
+
1914
+ ##
1915
+ # Add a column to the column family to expose in the table schema
1916
+ # that is specified as the `INTEGER` type.
1917
+ #
1918
+ # @param [String] qualifier Qualifier of the column. See
1919
+ # {BigtableSource::Column#qualifier}.
1920
+ # @param [String] as A valid identifier to be used as the column
1921
+ # field name if the qualifier is not a valid BigQuery field
1922
+ # identifier (i.e. does not match `[a-zA-Z][a-zA-Z0-9_]*`). See
1923
+ # {BigtableSource::Column#field_name}.
1924
+ #
1925
+ # @yield [column] a block for setting the column
1926
+ # @yieldparam [BigtableSource::Column] column the column object
1927
+ #
1928
+ # @return [Array<BigtableSource::Column>]
1929
+ #
1930
+ # @example
1931
+ # require "google/cloud/bigquery"
1932
+ #
1933
+ # bigquery = Google::Cloud::Bigquery.new
1934
+ #
1935
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1936
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1937
+ # bt.rowkey_as_string = true
1938
+ # bt.add_family "user" do |u|
1939
+ # u.add_integer "age"
1940
+ # end
1941
+ # end
1942
+ #
1943
+ def add_integer qualifier, as: nil
1944
+ col = add_column qualifier, as: as, type: "INTEGER"
1945
+ yield col if block_given?
1946
+ col
1947
+ end
1948
+
1949
+ ##
1950
+ # Add a column to the column family to expose in the table schema
1951
+ # that is specified as the `FLOAT` type.
1952
+ #
1953
+ # @param [String] qualifier Qualifier of the column. See
1954
+ # {BigtableSource::Column#qualifier}.
1955
+ # @param [String] as A valid identifier to be used as the column
1956
+ # field name if the qualifier is not a valid BigQuery field
1957
+ # identifier (i.e. does not match `[a-zA-Z][a-zA-Z0-9_]*`). See
1958
+ # {BigtableSource::Column#field_name}.
1959
+ #
1960
+ # @yield [column] a block for setting the column
1961
+ # @yieldparam [BigtableSource::Column] column the column object
1962
+ #
1963
+ # @return [Array<BigtableSource::Column>]
1964
+ #
1965
+ # @example
1966
+ # require "google/cloud/bigquery"
1967
+ #
1968
+ # bigquery = Google::Cloud::Bigquery.new
1969
+ #
1970
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
1971
+ # bigtable_table = bigquery.external bigtable_url do |bt|
1972
+ # bt.rowkey_as_string = true
1973
+ # bt.add_family "user" do |u|
1974
+ # u.add_float "score"
1975
+ # end
1976
+ # end
1977
+ #
1978
+ def add_float qualifier, as: nil
1979
+ col = add_column qualifier, as: as, type: "FLOAT"
1980
+ yield col if block_given?
1981
+ col
1982
+ end
1983
+
1984
+ ##
1985
+ # Add a column to the column family to expose in the table schema
1986
+ # that is specified as the `BOOLEAN` type.
1987
+ #
1988
+ # @param [String] qualifier Qualifier of the column. See
1989
+ # {BigtableSource::Column#qualifier}.
1990
+ # @param [String] as A valid identifier to be used as the column
1991
+ # field name if the qualifier is not a valid BigQuery field
1992
+ # identifier (i.e. does not match `[a-zA-Z][a-zA-Z0-9_]*`). See
1993
+ # {BigtableSource::Column#field_name}.
1994
+ #
1995
+ # @yield [column] a block for setting the column
1996
+ # @yieldparam [BigtableSource::Column] column the column object
1997
+ #
1998
+ # @return [Array<BigtableSource::Column>]
1999
+ #
2000
+ # @example
2001
+ # require "google/cloud/bigquery"
2002
+ #
2003
+ # bigquery = Google::Cloud::Bigquery.new
2004
+ #
2005
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
2006
+ # bigtable_table = bigquery.external bigtable_url do |bt|
2007
+ # bt.rowkey_as_string = true
2008
+ # bt.add_family "user" do |u|
2009
+ # u.add_boolean "active"
2010
+ # end
2011
+ # end
2012
+ #
2013
+ def add_boolean qualifier, as: nil
2014
+ col = add_column qualifier, as: as, type: "BOOLEAN"
2015
+ yield col if block_given?
2016
+ col
2017
+ end
2018
+
2019
+ ##
2020
+ # @private Google API Client object.
2021
+ def to_gapi
2022
+ @gapi.columns = @columns.map(&:to_gapi)
2023
+ @gapi
2024
+ end
2025
+
2026
+ ##
2027
+ # @private Google API Client object.
2028
+ def self.from_gapi gapi
2029
+ new_fam = new
2030
+ new_fam.instance_variable_set :@gapi, gapi
2031
+ columns = Array(gapi.columns).map { |col_gapi| BigtableSource::Column.from_gapi col_gapi }
2032
+ new_fam.instance_variable_set :@columns, columns
2033
+ new_fam
2034
+ end
2035
+
2036
+ ##
2037
+ # @private
2038
+ def freeze
2039
+ @columns.map(&:freeze!)
2040
+ @columns.freeze!
2041
+ super
2042
+ end
2043
+
2044
+ protected
2045
+
2046
+ def frozen_check!
2047
+ return unless frozen?
2048
+ raise ArgumentError, "Cannot modify external data source when frozen"
2049
+ end
2050
+ end
2051
+
2052
+ ##
2053
+ # # BigtableSource::Column
2054
+ #
2055
+ # A Bigtable column to expose in the table schema along with its
2056
+ # types.
2057
+ #
2058
+ # @example
2059
+ # require "google/cloud/bigquery"
2060
+ #
2061
+ # bigquery = Google::Cloud::Bigquery.new
2062
+ #
2063
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
2064
+ # bigtable_table = bigquery.external bigtable_url do |bt|
2065
+ # bt.rowkey_as_string = true
2066
+ # bt.add_family "user" do |u|
2067
+ # u.add_string "name"
2068
+ # u.add_string "email"
2069
+ # u.add_integer "age"
2070
+ # u.add_boolean "active"
2071
+ # end
2072
+ # end
2073
+ #
2074
+ # data = bigquery.query "SELECT * FROM my_ext_table",
2075
+ # external: { my_ext_table: bigtable_table }
2076
+ #
2077
+ # # Iterate over the first page of results
2078
+ # data.each do |row|
2079
+ # puts row[:name]
2080
+ # end
2081
+ # # Retrieve the next page of results
2082
+ # data = data.next if data.next?
2083
+ #
2084
+ class Column
2085
+ ##
2086
+ # @private Create an empty BigtableSource::Column object.
2087
+ def initialize
2088
+ @gapi = Google::Apis::BigqueryV2::BigtableColumn.new
2089
+ end
2090
+
2091
+ ##
2092
+ # Qualifier of the column. Columns in the parent column family that
2093
+ # has this exact qualifier are exposed as `.` field. If the
2094
+ # qualifier is valid UTF-8 string, it will be represented as a UTF-8
2095
+ # string. Otherwise, it will represented as a ASCII-8BIT string.
2096
+ #
2097
+ # If the qualifier is not a valid BigQuery field identifier (does
2098
+ # not match `[a-zA-Z][a-zA-Z0-9_]*`) a valid identifier must be
2099
+ # provided as `field_name`.
2100
+ #
2101
+ # @return [String]
2102
+ #
2103
+ # @example
2104
+ # require "google/cloud/bigquery"
2105
+ #
2106
+ # bigquery = Google::Cloud::Bigquery.new
2107
+ #
2108
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
2109
+ # bigtable_table = bigquery.external bigtable_url do |bt|
2110
+ # bt.add_family "user" do |u|
2111
+ # u.add_string "name" do |col|
2112
+ # col.qualifier # "user"
2113
+ # col.qualifier = "User"
2114
+ # col.qualifier # "User"
2115
+ # end
2116
+ # end
2117
+ # end
2118
+ #
2119
+ def qualifier
2120
+ @gapi.qualifier_string || Base64.strict_decode64(@gapi.qualifier_encoded.to_s)
2121
+ end
2122
+
2123
+ ##
2124
+ # Set the qualifier of the column. Columns in the parent column
2125
+ # family that has this exact qualifier are exposed as `.` field.
2126
+ # Values that are valid UTF-8 strings will be treated as such. All
2127
+ # other values will be treated as `BINARY`.
2128
+ #
2129
+ # @param [String] new_qualifier New qualifier value
2130
+ #
2131
+ # @example
2132
+ # require "google/cloud/bigquery"
2133
+ #
2134
+ # bigquery = Google::Cloud::Bigquery.new
2135
+ #
2136
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
2137
+ # bigtable_table = bigquery.external bigtable_url do |bt|
2138
+ # bt.add_family "user" do |u|
2139
+ # u.add_string "name" do |col|
2140
+ # col.qualifier # "user"
2141
+ # col.qualifier = "User"
2142
+ # col.qualifier # "User"
2143
+ # end
2144
+ # end
2145
+ # end
2146
+ #
2147
+ def qualifier= new_qualifier
2148
+ frozen_check!
2149
+ raise ArgumentError if new_qualifier.nil?
2150
+
2151
+ utf8_qualifier = new_qualifier.encode Encoding::UTF_8
2152
+ if utf8_qualifier.valid_encoding?
2153
+ @gapi.qualifier_string = utf8_qualifier
2154
+ if @gapi.instance_variables.include? :@qualifier_encoded
2155
+ @gapi.remove_instance_variable :@qualifier_encoded
2156
+ end
2157
+ else
2158
+ @gapi.qualifier_encoded = Base64.strict_encode64 new_qualifier
2159
+ if @gapi.instance_variables.include? :@qualifier_string
2160
+ @gapi.remove_instance_variable :@qualifier_string
2161
+ end
2162
+ end
2163
+ rescue EncodingError
2164
+ @gapi.qualifier_encoded = Base64.strict_encode64 new_qualifier
2165
+ @gapi.remove_instance_variable :@qualifier_string if @gapi.instance_variables.include? :@qualifier_string
2166
+ end
2167
+
2168
+ ##
2169
+ # The encoding of the values when the type is not `STRING`.
2170
+ #
2171
+ # @return [String]
2172
+ #
2173
+ # @example
2174
+ # require "google/cloud/bigquery"
2175
+ #
2176
+ # bigquery = Google::Cloud::Bigquery.new
2177
+ #
2178
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
2179
+ # bigtable_table = bigquery.external bigtable_url do |bt|
2180
+ # bt.add_family "user" do |u|
2181
+ # u.add_bytes "name" do |col|
2182
+ # col.encoding = "TEXT"
2183
+ # col.encoding # "TEXT"
2184
+ # end
2185
+ # end
2186
+ # end
2187
+ #
2188
+ def encoding
2189
+ @gapi.encoding
2190
+ end
2191
+
2192
+ ##
2193
+ # Set the encoding of the values when the type is not `STRING`.
2194
+ # Acceptable encoding values are:
2195
+ #
2196
+ # * `TEXT` - indicates values are alphanumeric text strings.
2197
+ # * `BINARY` - indicates values are encoded using HBase
2198
+ # `Bytes.toBytes` family of functions. This can be overridden on a
2199
+ # column.
2200
+ #
2201
+ # @param [String] new_encoding New encoding value
2202
+ #
2203
+ # @example
2204
+ # require "google/cloud/bigquery"
2205
+ #
2206
+ # bigquery = Google::Cloud::Bigquery.new
2207
+ #
2208
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
2209
+ # bigtable_table = bigquery.external bigtable_url do |bt|
2210
+ # bt.add_family "user" do |u|
2211
+ # u.add_bytes "name" do |col|
2212
+ # col.encoding = "TEXT"
2213
+ # col.encoding # "TEXT"
2214
+ # end
2215
+ # end
2216
+ # end
2217
+ #
2218
+ def encoding= new_encoding
2219
+ frozen_check!
2220
+ @gapi.encoding = new_encoding
2221
+ end
2222
+
2223
+ ##
2224
+ # If the qualifier is not a valid BigQuery field identifier (does
2225
+ # not match `[a-zA-Z][a-zA-Z0-9_]*`) a valid identifier must be
2226
+ # provided as the column field name and is used as field name in
2227
+ # queries.
2228
+ #
2229
+ # @return [String]
2230
+ #
2231
+ # @example
2232
+ # require "google/cloud/bigquery"
2233
+ #
2234
+ # bigquery = Google::Cloud::Bigquery.new
2235
+ #
2236
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
2237
+ # bigtable_table = bigquery.external bigtable_url do |bt|
2238
+ # bt.add_family "user" do |u|
2239
+ # u.add_string "001_name", as: "user" do |col|
2240
+ # col.field_name # "user"
2241
+ # col.field_name = "User"
2242
+ # col.field_name # "User"
2243
+ # end
2244
+ # end
2245
+ # end
2246
+ #
2247
+ def field_name
2248
+ @gapi.field_name
2249
+ end
2250
+
2251
+ ##
2252
+ # Sets the identifier to be used as the column field name in queries
2253
+ # when the qualifier is not a valid BigQuery field identifier (does
2254
+ # not match `[a-zA-Z][a-zA-Z0-9_]*`).
2255
+ #
2256
+ # @param [String] new_field_name New field_name value
2257
+ #
2258
+ # @example
2259
+ # require "google/cloud/bigquery"
2260
+ #
2261
+ # bigquery = Google::Cloud::Bigquery.new
2262
+ #
2263
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
2264
+ # bigtable_table = bigquery.external bigtable_url do |bt|
2265
+ # bt.add_family "user" do |u|
2266
+ # u.add_string "001_name", as: "user" do |col|
2267
+ # col.field_name # "user"
2268
+ # col.field_name = "User"
2269
+ # col.field_name # "User"
2270
+ # end
2271
+ # end
2272
+ # end
2273
+ #
2274
+ def field_name= new_field_name
2275
+ frozen_check!
2276
+ @gapi.field_name = new_field_name
2277
+ end
2278
+
2279
+ ##
2280
+ # Whether only the latest version of value in this column are
2281
+ # exposed. Can also be set at the column family level. However, this
2282
+ # value takes precedence when set at both levels.
2283
+ #
2284
+ # @return [Boolean]
2285
+ #
2286
+ # @example
2287
+ # require "google/cloud/bigquery"
2288
+ #
2289
+ # bigquery = Google::Cloud::Bigquery.new
2290
+ #
2291
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
2292
+ # bigtable_table = bigquery.external bigtable_url do |bt|
2293
+ # bt.add_family "user" do |u|
2294
+ # u.add_string "name" do |col|
2295
+ # col.latest = true
2296
+ # col.latest # true
2297
+ # end
2298
+ # end
2299
+ # end
2300
+ #
2301
+ def latest
2302
+ @gapi.only_read_latest
2303
+ end
2304
+
2305
+ ##
2306
+ # Set whether only the latest version of value in this column are
2307
+ # exposed. Can also be set at the column family level. However, this
2308
+ # value takes precedence when set at both levels.
2309
+ #
2310
+ # @param [Boolean] new_latest New latest value
2311
+ #
2312
+ # @example
2313
+ # require "google/cloud/bigquery"
2314
+ #
2315
+ # bigquery = Google::Cloud::Bigquery.new
2316
+ #
2317
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
2318
+ # bigtable_table = bigquery.external bigtable_url do |bt|
2319
+ # bt.add_family "user" do |u|
2320
+ # u.add_string "name" do |col|
2321
+ # col.latest = true
2322
+ # col.latest # true
2323
+ # end
2324
+ # end
2325
+ # end
2326
+ #
2327
+ def latest= new_latest
2328
+ frozen_check!
2329
+ @gapi.only_read_latest = new_latest
2330
+ end
2331
+
2332
+ ##
2333
+ # The type to convert the value in cells of this column. The values
2334
+ # are expected to be encoded using HBase `Bytes.toBytes` function
2335
+ # when using the `BINARY` encoding value. The following BigQuery
2336
+ # types are allowed:
2337
+ #
2338
+ # * `BYTES`
2339
+ # * `STRING`
2340
+ # * `INTEGER`
2341
+ # * `FLOAT`
2342
+ # * `BOOLEAN`
2343
+ #
2344
+ # Default type is `BYTES`. Can also be set at the column family
2345
+ # level. However, this value takes precedence when set at both
2346
+ # levels.
2347
+ #
2348
+ # @return [String]
2349
+ #
2350
+ # @example
2351
+ # require "google/cloud/bigquery"
2352
+ #
2353
+ # bigquery = Google::Cloud::Bigquery.new
2354
+ #
2355
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
2356
+ # bigtable_table = bigquery.external bigtable_url do |bt|
2357
+ # bt.add_family "user" do |u|
2358
+ # u.add_string "name" do |col|
2359
+ # col.type # "STRING"
2360
+ # end
2361
+ # end
2362
+ # end
2363
+ #
2364
+ def type
2365
+ @gapi.type
2366
+ end
2367
+
2368
+ ##
2369
+ # Set the type to convert the value in cells of this column. The
2370
+ # values are expected to be encoded using HBase `Bytes.toBytes`
2371
+ # function when using the `BINARY` encoding value. The following
2372
+ # BigQuery types are allowed:
2373
+ #
2374
+ # * `BYTES`
2375
+ # * `STRING`
2376
+ # * `INTEGER`
2377
+ # * `FLOAT`
2378
+ # * `BOOLEAN`
2379
+ #
2380
+ # Default type is `BYTES`. Can also be set at the column family
2381
+ # level. However, this value takes precedence when set at both
2382
+ # levels.
2383
+ #
2384
+ # @param [String] new_type New type value
2385
+ #
2386
+ # @example
2387
+ # require "google/cloud/bigquery"
2388
+ #
2389
+ # bigquery = Google::Cloud::Bigquery.new
2390
+ #
2391
+ # bigtable_url = "https://googleapis.com/bigtable/projects/..."
2392
+ # bigtable_table = bigquery.external bigtable_url do |bt|
2393
+ # bt.add_family "user" do |u|
2394
+ # u.add_string "name" do |col|
2395
+ # col.type # "STRING"
2396
+ # col.type = "BYTES"
2397
+ # col.type # "BYTES"
2398
+ # end
2399
+ # end
2400
+ # end
2401
+ #
2402
+ def type= new_type
2403
+ frozen_check!
2404
+ @gapi.type = new_type
2405
+ end
2406
+
2407
+ ##
2408
+ # @private Google API Client object.
2409
+ def to_gapi
2410
+ @gapi
2411
+ end
2412
+
2413
+ ##
2414
+ # @private Google API Client object.
2415
+ def self.from_gapi gapi
2416
+ new_col = new
2417
+ new_col.instance_variable_set :@gapi, gapi
2418
+ new_col
2419
+ end
2420
+
2421
+ protected
2422
+
2423
+ def frozen_check!
2424
+ return unless frozen?
2425
+ raise ArgumentError, "Cannot modify external data source when frozen"
2426
+ end
2427
+ end
2428
+ end
2429
+ end
2430
+ end
2431
+ end
2432
+ end