google-cloud-bigquery 1.21.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.yardopts +16 -0
- data/AUTHENTICATION.md +158 -0
- data/CHANGELOG.md +397 -0
- data/CODE_OF_CONDUCT.md +40 -0
- data/CONTRIBUTING.md +188 -0
- data/LICENSE +201 -0
- data/LOGGING.md +27 -0
- data/OVERVIEW.md +463 -0
- data/TROUBLESHOOTING.md +31 -0
- data/lib/google-cloud-bigquery.rb +139 -0
- data/lib/google/cloud/bigquery.rb +145 -0
- data/lib/google/cloud/bigquery/argument.rb +197 -0
- data/lib/google/cloud/bigquery/convert.rb +383 -0
- data/lib/google/cloud/bigquery/copy_job.rb +316 -0
- data/lib/google/cloud/bigquery/credentials.rb +50 -0
- data/lib/google/cloud/bigquery/data.rb +526 -0
- data/lib/google/cloud/bigquery/dataset.rb +2845 -0
- data/lib/google/cloud/bigquery/dataset/access.rb +1021 -0
- data/lib/google/cloud/bigquery/dataset/list.rb +162 -0
- data/lib/google/cloud/bigquery/encryption_configuration.rb +123 -0
- data/lib/google/cloud/bigquery/external.rb +2432 -0
- data/lib/google/cloud/bigquery/extract_job.rb +368 -0
- data/lib/google/cloud/bigquery/insert_response.rb +180 -0
- data/lib/google/cloud/bigquery/job.rb +657 -0
- data/lib/google/cloud/bigquery/job/list.rb +162 -0
- data/lib/google/cloud/bigquery/load_job.rb +1704 -0
- data/lib/google/cloud/bigquery/model.rb +740 -0
- data/lib/google/cloud/bigquery/model/list.rb +164 -0
- data/lib/google/cloud/bigquery/project.rb +1655 -0
- data/lib/google/cloud/bigquery/project/list.rb +161 -0
- data/lib/google/cloud/bigquery/query_job.rb +1695 -0
- data/lib/google/cloud/bigquery/routine.rb +1108 -0
- data/lib/google/cloud/bigquery/routine/list.rb +165 -0
- data/lib/google/cloud/bigquery/schema.rb +564 -0
- data/lib/google/cloud/bigquery/schema/field.rb +668 -0
- data/lib/google/cloud/bigquery/service.rb +589 -0
- data/lib/google/cloud/bigquery/standard_sql.rb +495 -0
- data/lib/google/cloud/bigquery/table.rb +3340 -0
- data/lib/google/cloud/bigquery/table/async_inserter.rb +520 -0
- data/lib/google/cloud/bigquery/table/list.rb +172 -0
- data/lib/google/cloud/bigquery/time.rb +65 -0
- data/lib/google/cloud/bigquery/version.rb +22 -0
- metadata +297 -0
@@ -0,0 +1,162 @@
|
|
1
|
+
# Copyright 2015 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
require "delegate"
|
17
|
+
|
18
|
+
module Google
|
19
|
+
module Cloud
|
20
|
+
module Bigquery
|
21
|
+
class Dataset
|
22
|
+
##
|
23
|
+
# Dataset::List is a special case Array with additional values.
|
24
|
+
class List < DelegateClass(::Array)
|
25
|
+
##
|
26
|
+
# If not empty, indicates that there are more records that match
|
27
|
+
# the request and this value should be passed to continue.
|
28
|
+
attr_accessor :token
|
29
|
+
|
30
|
+
# A hash of this page of results.
|
31
|
+
attr_accessor :etag
|
32
|
+
|
33
|
+
##
|
34
|
+
# @private Create a new Dataset::List with an array of datasets.
|
35
|
+
def initialize arr = []
|
36
|
+
super arr
|
37
|
+
end
|
38
|
+
|
39
|
+
##
|
40
|
+
# Whether there is a next page of datasets.
|
41
|
+
#
|
42
|
+
# @return [Boolean]
|
43
|
+
#
|
44
|
+
# @example
|
45
|
+
# require "google/cloud/bigquery"
|
46
|
+
#
|
47
|
+
# bigquery = Google::Cloud::Bigquery.new
|
48
|
+
#
|
49
|
+
# datasets = bigquery.datasets
|
50
|
+
# if datasets.next?
|
51
|
+
# next_datasets = datasets.next
|
52
|
+
# end
|
53
|
+
def next?
|
54
|
+
!token.nil?
|
55
|
+
end
|
56
|
+
|
57
|
+
##
|
58
|
+
# Retrieve the next page of datasets.
|
59
|
+
#
|
60
|
+
# @return [Dataset::List]
|
61
|
+
#
|
62
|
+
# @example
|
63
|
+
# require "google/cloud/bigquery"
|
64
|
+
#
|
65
|
+
# bigquery = Google::Cloud::Bigquery.new
|
66
|
+
#
|
67
|
+
# datasets = bigquery.datasets
|
68
|
+
# if datasets.next?
|
69
|
+
# next_datasets = datasets.next
|
70
|
+
# end
|
71
|
+
def next
|
72
|
+
return nil unless next?
|
73
|
+
ensure_service!
|
74
|
+
gapi = @service.list_datasets all: @hidden, filter: @filter, token: token, max: @max
|
75
|
+
self.class.from_gapi gapi, @service, @hidden, @filter, @max
|
76
|
+
end
|
77
|
+
|
78
|
+
##
|
79
|
+
# Retrieves remaining results by repeatedly invoking {#next} until
|
80
|
+
# {#next?} returns `false`. Calls the given block once for each
|
81
|
+
# result, which is passed as the argument to the block.
|
82
|
+
#
|
83
|
+
# An Enumerator is returned if no block is given.
|
84
|
+
#
|
85
|
+
# This method will make repeated API calls until all remaining results
|
86
|
+
# are retrieved. (Unlike `#each`, for example, which merely iterates
|
87
|
+
# over the results returned by a single API call.) Use with caution.
|
88
|
+
#
|
89
|
+
# @param [Integer] request_limit The upper limit of API requests to
|
90
|
+
# make to load all datasets. Default is no limit.
|
91
|
+
# @yield [dataset] The block for accessing each dataset.
|
92
|
+
# @yieldparam [Dataset] dataset The dataset object.
|
93
|
+
#
|
94
|
+
# @return [Enumerator]
|
95
|
+
#
|
96
|
+
# @example Iterating each result by passing a block:
|
97
|
+
# require "google/cloud/bigquery"
|
98
|
+
#
|
99
|
+
# bigquery = Google::Cloud::Bigquery.new
|
100
|
+
#
|
101
|
+
# bigquery.datasets.all do |dataset|
|
102
|
+
# puts dataset.name
|
103
|
+
# end
|
104
|
+
#
|
105
|
+
# @example Using the enumerator by not passing a block:
|
106
|
+
# require "google/cloud/bigquery"
|
107
|
+
#
|
108
|
+
# bigquery = Google::Cloud::Bigquery.new
|
109
|
+
#
|
110
|
+
# all_names = bigquery.datasets.all.map do |dataset|
|
111
|
+
# dataset.name
|
112
|
+
# end
|
113
|
+
#
|
114
|
+
# @example Limit the number of API calls made:
|
115
|
+
# require "google/cloud/bigquery"
|
116
|
+
#
|
117
|
+
# bigquery = Google::Cloud::Bigquery.new
|
118
|
+
#
|
119
|
+
# bigquery.datasets.all(request_limit: 10) do |dataset|
|
120
|
+
# puts dataset.name
|
121
|
+
# end
|
122
|
+
#
|
123
|
+
def all request_limit: nil
|
124
|
+
request_limit = request_limit.to_i if request_limit
|
125
|
+
return enum_for :all, request_limit: request_limit unless block_given?
|
126
|
+
results = self
|
127
|
+
loop do
|
128
|
+
results.each { |r| yield r }
|
129
|
+
if request_limit
|
130
|
+
request_limit -= 1
|
131
|
+
break if request_limit.negative?
|
132
|
+
end
|
133
|
+
break unless results.next?
|
134
|
+
results = results.next
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
##
|
139
|
+
# @private New Dataset::List from a response object.
|
140
|
+
def self.from_gapi gapi_list, service, hidden = nil, filter = nil, max = nil
|
141
|
+
datasets = List.new(Array(gapi_list.datasets).map { |gapi_object| Dataset.from_gapi gapi_object, service })
|
142
|
+
datasets.instance_variable_set :@token, gapi_list.next_page_token
|
143
|
+
datasets.instance_variable_set :@etag, gapi_list.etag
|
144
|
+
datasets.instance_variable_set :@service, service
|
145
|
+
datasets.instance_variable_set :@hidden, hidden
|
146
|
+
datasets.instance_variable_set :@filter, filter
|
147
|
+
datasets.instance_variable_set :@max, max
|
148
|
+
datasets
|
149
|
+
end
|
150
|
+
|
151
|
+
protected
|
152
|
+
|
153
|
+
##
|
154
|
+
# Raise an error unless an active service is available.
|
155
|
+
def ensure_service!
|
156
|
+
raise "Must have active connection" unless @service
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
@@ -0,0 +1,123 @@
|
|
1
|
+
# Copyright 2018 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require "google/apis/bigquery_v2"
|
16
|
+
|
17
|
+
module Google
|
18
|
+
module Cloud
|
19
|
+
module Bigquery
|
20
|
+
##
|
21
|
+
# # Encryption Configuration
|
22
|
+
#
|
23
|
+
# A builder for BigQuery table encryption configurations, passed to block
|
24
|
+
# arguments to {Dataset#create_table} and {Table#encryption}.
|
25
|
+
#
|
26
|
+
# @see https://cloud.google.com/bigquery/docs/customer-managed-encryption
|
27
|
+
# Protecting Data with Cloud KMS Keys
|
28
|
+
#
|
29
|
+
# @example
|
30
|
+
# require "google/cloud/bigquery"
|
31
|
+
#
|
32
|
+
# bigquery = Google::Cloud::Bigquery.new
|
33
|
+
# dataset = bigquery.dataset "my_dataset"
|
34
|
+
# key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
|
35
|
+
# encrypt_config = bigquery.encryption kms_key: key_name
|
36
|
+
# table = dataset.create_table "my_table" do |updater|
|
37
|
+
# updater.encryption = encrypt_config
|
38
|
+
# end
|
39
|
+
#
|
40
|
+
class EncryptionConfiguration
|
41
|
+
##
|
42
|
+
# @private The Google API Client object.
|
43
|
+
attr_accessor :gapi
|
44
|
+
|
45
|
+
##
|
46
|
+
# @private Create an empty EncryptionConfiguration object.
|
47
|
+
def initialize
|
48
|
+
@gapi = Google::Apis::BigqueryV2::EncryptionConfiguration.new
|
49
|
+
end
|
50
|
+
|
51
|
+
##
|
52
|
+
# The Cloud KMS encryption key that will be used to protect the table.
|
53
|
+
# For example: `projects/a/locations/b/keyRings/c/cryptoKeys/d`
|
54
|
+
# The default value is `nil`, which means default encryption is used.
|
55
|
+
#
|
56
|
+
# @return [String]
|
57
|
+
#
|
58
|
+
# @example
|
59
|
+
# require "google/cloud/bigquery"
|
60
|
+
#
|
61
|
+
# config = Google::Cloud::Bigquery::EncryptionConfiguration.new
|
62
|
+
# key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
|
63
|
+
# config.kms_key = key_name
|
64
|
+
#
|
65
|
+
def kms_key
|
66
|
+
@gapi.kms_key_name
|
67
|
+
end
|
68
|
+
|
69
|
+
##
|
70
|
+
# Set the Cloud KMS encryption key that will be used to protect the
|
71
|
+
# table. For example: `projects/a/locations/b/keyRings/c/cryptoKeys/d`
|
72
|
+
# The default value is `nil`, which means default encryption is used.
|
73
|
+
#
|
74
|
+
# @param [String] new_kms_key_name New Cloud KMS key name
|
75
|
+
#
|
76
|
+
# @example
|
77
|
+
# require "google/cloud/bigquery"
|
78
|
+
#
|
79
|
+
# config = Google::Cloud::Bigquery::EncryptionConfiguration.new
|
80
|
+
# key_name = "projects/a/locations/b/keyRings/c/cryptoKeys/d"
|
81
|
+
# config.kms_key = key_name
|
82
|
+
#
|
83
|
+
def kms_key= new_kms_key_name
|
84
|
+
frozen_check!
|
85
|
+
@gapi.kms_key_name = new_kms_key_name
|
86
|
+
end
|
87
|
+
|
88
|
+
# @private
|
89
|
+
def changed?
|
90
|
+
return false if frozen?
|
91
|
+
@original_json != @gapi.to_json
|
92
|
+
end
|
93
|
+
|
94
|
+
##
|
95
|
+
# @private Google API Client object.
|
96
|
+
def to_gapi
|
97
|
+
@gapi
|
98
|
+
end
|
99
|
+
|
100
|
+
##
|
101
|
+
# @private Google API Client object.
|
102
|
+
def self.from_gapi gapi
|
103
|
+
new_config = new
|
104
|
+
new_config.instance_variable_set :@gapi, gapi
|
105
|
+
new_config
|
106
|
+
end
|
107
|
+
|
108
|
+
# @private
|
109
|
+
def == other
|
110
|
+
return false unless other.is_a? EncryptionConfiguration
|
111
|
+
to_gapi.to_json == other.to_gapi.to_json
|
112
|
+
end
|
113
|
+
|
114
|
+
protected
|
115
|
+
|
116
|
+
def frozen_check!
|
117
|
+
return unless frozen?
|
118
|
+
raise ArgumentError, "Cannot modify a frozen encryption configuration"
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
@@ -0,0 +1,2432 @@
|
|
1
|
+
# Copyright 2017 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
require "google/apis/bigquery_v2"
|
17
|
+
require "base64"
|
18
|
+
|
19
|
+
module Google
|
20
|
+
module Cloud
|
21
|
+
module Bigquery
|
22
|
+
##
|
23
|
+
# # External
|
24
|
+
#
|
25
|
+
# Creates a new {External::DataSource} (or subclass) object that
|
26
|
+
# represents the external data source that can be queried from directly,
|
27
|
+
# even though the data is not stored in BigQuery. Instead of loading or
|
28
|
+
# streaming the data, this object references the external data source.
|
29
|
+
#
|
30
|
+
# See {External::DataSource}, {External::CsvSource},
|
31
|
+
# {External::JsonSource}, {External::SheetsSource},
|
32
|
+
# {External::BigtableSource}
|
33
|
+
#
|
34
|
+
# @example
|
35
|
+
# require "google/cloud/bigquery"
|
36
|
+
#
|
37
|
+
# bigquery = Google::Cloud::Bigquery.new
|
38
|
+
#
|
39
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
40
|
+
# csv_table = bigquery.external csv_url do |csv|
|
41
|
+
# csv.autodetect = true
|
42
|
+
# csv.skip_leading_rows = 1
|
43
|
+
# end
|
44
|
+
#
|
45
|
+
# data = bigquery.query "SELECT * FROM my_ext_table",
|
46
|
+
# external: { my_ext_table: csv_table }
|
47
|
+
#
|
48
|
+
# # Iterate over the first page of results
|
49
|
+
# data.each do |row|
|
50
|
+
# puts row[:name]
|
51
|
+
# end
|
52
|
+
# # Retrieve the next page of results
|
53
|
+
# data = data.next if data.next?
|
54
|
+
#
|
55
|
+
module External
|
56
|
+
##
|
57
|
+
# @private New External from URLs and format
|
58
|
+
def self.from_urls urls, format = nil
|
59
|
+
external_format = source_format_for urls, format
|
60
|
+
raise ArgumentError, "Unable to determine external table format" if external_format.nil?
|
61
|
+
external_class = table_class_for external_format
|
62
|
+
external_class.new.tap do |e|
|
63
|
+
e.gapi.source_uris = Array(urls)
|
64
|
+
e.gapi.source_format = external_format
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
##
|
69
|
+
# @private Google API Client object.
|
70
|
+
def self.from_gapi gapi
|
71
|
+
external_format = source_format_for gapi.source_uris,
|
72
|
+
gapi.source_format
|
73
|
+
raise ArgumentError, "Unable to determine external table format" if external_format.nil?
|
74
|
+
external_class = table_class_for external_format
|
75
|
+
external_class.from_gapi gapi
|
76
|
+
end
|
77
|
+
|
78
|
+
##
|
79
|
+
# @private Determine source_format from inputs
|
80
|
+
def self.source_format_for urls, format
|
81
|
+
val = {
|
82
|
+
"csv" => "CSV", "avro" => "AVRO",
|
83
|
+
"json" => "NEWLINE_DELIMITED_JSON",
|
84
|
+
"newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
|
85
|
+
"sheets" => "GOOGLE_SHEETS",
|
86
|
+
"google_sheets" => "GOOGLE_SHEETS",
|
87
|
+
"datastore" => "DATASTORE_BACKUP",
|
88
|
+
"backup" => "DATASTORE_BACKUP",
|
89
|
+
"datastore_backup" => "DATASTORE_BACKUP",
|
90
|
+
"bigtable" => "BIGTABLE"
|
91
|
+
}[format.to_s.downcase]
|
92
|
+
return val unless val.nil?
|
93
|
+
Array(urls).each do |url|
|
94
|
+
return "CSV" if url.end_with? ".csv"
|
95
|
+
return "NEWLINE_DELIMITED_JSON" if url.end_with? ".json"
|
96
|
+
return "AVRO" if url.end_with? ".avro"
|
97
|
+
return "DATASTORE_BACKUP" if url.end_with? ".backup_info"
|
98
|
+
return "GOOGLE_SHEETS" if url.start_with? "https://docs.google.com/spreadsheets/"
|
99
|
+
return "BIGTABLE" if url.start_with? "https://googleapis.com/bigtable/projects/"
|
100
|
+
end
|
101
|
+
nil
|
102
|
+
end
|
103
|
+
|
104
|
+
##
|
105
|
+
# @private Determine table class from source_format
|
106
|
+
def self.table_class_for format
|
107
|
+
case format
|
108
|
+
when "CSV" then External::CsvSource
|
109
|
+
when "NEWLINE_DELIMITED_JSON" then External::JsonSource
|
110
|
+
when "GOOGLE_SHEETS" then External::SheetsSource
|
111
|
+
when "BIGTABLE" then External::BigtableSource
|
112
|
+
else
|
113
|
+
# AVRO and DATASTORE_BACKUP
|
114
|
+
External::DataSource
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
##
|
119
|
+
# # DataSource
|
120
|
+
#
|
121
|
+
# External::DataSource and its subclasses represents an external data
|
122
|
+
# source that can be queried from directly, even though the data is not
|
123
|
+
# stored in BigQuery. Instead of loading or streaming the data, this
|
124
|
+
# object references the external data source.
|
125
|
+
#
|
126
|
+
# The AVRO and Datastore Backup formats use {External::DataSource}. See
|
127
|
+
# {External::CsvSource}, {External::JsonSource},
|
128
|
+
# {External::SheetsSource}, {External::BigtableSource} for the other
|
129
|
+
# formats.
|
130
|
+
#
|
131
|
+
# @example
|
132
|
+
# require "google/cloud/bigquery"
|
133
|
+
#
|
134
|
+
# bigquery = Google::Cloud::Bigquery.new
|
135
|
+
#
|
136
|
+
# avro_url = "gs://bucket/path/to/data.avro"
|
137
|
+
# avro_table = bigquery.external avro_url do |avro|
|
138
|
+
# avro.autodetect = true
|
139
|
+
# end
|
140
|
+
#
|
141
|
+
# data = bigquery.query "SELECT * FROM my_ext_table",
|
142
|
+
# external: { my_ext_table: avro_table }
|
143
|
+
#
|
144
|
+
# # Iterate over the first page of results
|
145
|
+
# data.each do |row|
|
146
|
+
# puts row[:name]
|
147
|
+
# end
|
148
|
+
# # Retrieve the next page of results
|
149
|
+
# data = data.next if data.next?
|
150
|
+
#
|
151
|
+
class DataSource
|
152
|
+
##
|
153
|
+
# @private The Google API Client object.
|
154
|
+
attr_accessor :gapi
|
155
|
+
|
156
|
+
##
|
157
|
+
# @private Create an empty Table object.
|
158
|
+
def initialize
|
159
|
+
@gapi = Google::Apis::BigqueryV2::ExternalDataConfiguration.new
|
160
|
+
end
|
161
|
+
|
162
|
+
##
|
163
|
+
# The data format. For CSV files, specify "CSV". For Google sheets,
|
164
|
+
# specify "GOOGLE_SHEETS". For newline-delimited JSON, specify
|
165
|
+
# "NEWLINE_DELIMITED_JSON". For Avro files, specify "AVRO". For Google
|
166
|
+
# Cloud Datastore backups, specify "DATASTORE_BACKUP". [Beta] For
|
167
|
+
# Google Cloud Bigtable, specify "BIGTABLE".
|
168
|
+
#
|
169
|
+
# @return [String]
|
170
|
+
#
|
171
|
+
# @example
|
172
|
+
# require "google/cloud/bigquery"
|
173
|
+
#
|
174
|
+
# bigquery = Google::Cloud::Bigquery.new
|
175
|
+
#
|
176
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
177
|
+
# csv_table = bigquery.external csv_url
|
178
|
+
#
|
179
|
+
# csv_table.format #=> "CSV"
|
180
|
+
#
|
181
|
+
def format
|
182
|
+
@gapi.source_format
|
183
|
+
end
|
184
|
+
|
185
|
+
##
|
186
|
+
# Whether the data format is "CSV".
|
187
|
+
#
|
188
|
+
# @return [Boolean]
|
189
|
+
#
|
190
|
+
# @example
|
191
|
+
# require "google/cloud/bigquery"
|
192
|
+
#
|
193
|
+
# bigquery = Google::Cloud::Bigquery.new
|
194
|
+
#
|
195
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
196
|
+
# csv_table = bigquery.external csv_url
|
197
|
+
#
|
198
|
+
# csv_table.format #=> "CSV"
|
199
|
+
# csv_table.csv? #=> true
|
200
|
+
#
|
201
|
+
def csv?
|
202
|
+
@gapi.source_format == "CSV"
|
203
|
+
end
|
204
|
+
|
205
|
+
##
|
206
|
+
# Whether the data format is "NEWLINE_DELIMITED_JSON".
|
207
|
+
#
|
208
|
+
# @return [Boolean]
|
209
|
+
#
|
210
|
+
# @example
|
211
|
+
# require "google/cloud/bigquery"
|
212
|
+
#
|
213
|
+
# bigquery = Google::Cloud::Bigquery.new
|
214
|
+
#
|
215
|
+
# json_url = "gs://bucket/path/to/data.json"
|
216
|
+
# json_table = bigquery.external json_url
|
217
|
+
#
|
218
|
+
# json_table.format #=> "NEWLINE_DELIMITED_JSON"
|
219
|
+
# json_table.json? #=> true
|
220
|
+
#
|
221
|
+
def json?
|
222
|
+
@gapi.source_format == "NEWLINE_DELIMITED_JSON"
|
223
|
+
end
|
224
|
+
|
225
|
+
##
|
226
|
+
# Whether the data format is "GOOGLE_SHEETS".
|
227
|
+
#
|
228
|
+
# @return [Boolean]
|
229
|
+
#
|
230
|
+
# @example
|
231
|
+
# require "google/cloud/bigquery"
|
232
|
+
#
|
233
|
+
# bigquery = Google::Cloud::Bigquery.new
|
234
|
+
#
|
235
|
+
# sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
|
236
|
+
# sheets_table = bigquery.external sheets_url
|
237
|
+
#
|
238
|
+
# sheets_table.format #=> "GOOGLE_SHEETS"
|
239
|
+
# sheets_table.sheets? #=> true
|
240
|
+
#
|
241
|
+
def sheets?
|
242
|
+
@gapi.source_format == "GOOGLE_SHEETS"
|
243
|
+
end
|
244
|
+
|
245
|
+
##
|
246
|
+
# Whether the data format is "AVRO".
|
247
|
+
#
|
248
|
+
# @return [Boolean]
|
249
|
+
#
|
250
|
+
# @example
|
251
|
+
# require "google/cloud/bigquery"
|
252
|
+
#
|
253
|
+
# bigquery = Google::Cloud::Bigquery.new
|
254
|
+
#
|
255
|
+
# avro_url = "gs://bucket/path/to/data.avro"
|
256
|
+
# avro_table = bigquery.external avro_url
|
257
|
+
#
|
258
|
+
# avro_table.format #=> "AVRO"
|
259
|
+
# avro_table.avro? #=> true
|
260
|
+
#
|
261
|
+
def avro?
|
262
|
+
@gapi.source_format == "AVRO"
|
263
|
+
end
|
264
|
+
|
265
|
+
##
|
266
|
+
# Whether the data format is "DATASTORE_BACKUP".
|
267
|
+
#
|
268
|
+
# @return [Boolean]
|
269
|
+
#
|
270
|
+
# @example
|
271
|
+
# require "google/cloud/bigquery"
|
272
|
+
#
|
273
|
+
# bigquery = Google::Cloud::Bigquery.new
|
274
|
+
#
|
275
|
+
# backup_url = "gs://bucket/path/to/data.backup_info"
|
276
|
+
# backup_table = bigquery.external backup_url
|
277
|
+
#
|
278
|
+
# backup_table.format #=> "DATASTORE_BACKUP"
|
279
|
+
# backup_table.backup? #=> true
|
280
|
+
#
|
281
|
+
def backup?
|
282
|
+
@gapi.source_format == "DATASTORE_BACKUP"
|
283
|
+
end
|
284
|
+
|
285
|
+
##
|
286
|
+
# Whether the data format is "BIGTABLE".
|
287
|
+
#
|
288
|
+
# @return [Boolean]
|
289
|
+
#
|
290
|
+
# @example
|
291
|
+
# require "google/cloud/bigquery"
|
292
|
+
#
|
293
|
+
# bigquery = Google::Cloud::Bigquery.new
|
294
|
+
#
|
295
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
296
|
+
# bigtable_table = bigquery.external bigtable_url
|
297
|
+
#
|
298
|
+
# bigtable_table.format #=> "BIGTABLE"
|
299
|
+
# bigtable_table.bigtable? #=> true
|
300
|
+
#
|
301
|
+
def bigtable?
|
302
|
+
@gapi.source_format == "BIGTABLE"
|
303
|
+
end
|
304
|
+
|
305
|
+
##
|
306
|
+
# The fully-qualified URIs that point to your data in Google Cloud.
|
307
|
+
# For Google Cloud Storage URIs: Each URI can contain one '*' wildcard
|
308
|
+
# character and it must come after the 'bucket' name. Size limits
|
309
|
+
# related to load jobs apply to external data sources. For Google
|
310
|
+
# Cloud Bigtable URIs: Exactly one URI can be specified and it has be
|
311
|
+
# a fully specified and valid HTTPS URL for a Google Cloud Bigtable
|
312
|
+
# table. For Google Cloud Datastore backups, exactly one URI can be
|
313
|
+
# specified, and it must end with '.backup_info'. Also, the '*'
|
314
|
+
# wildcard character is not allowed.
|
315
|
+
#
|
316
|
+
# @return [Array<String>]
|
317
|
+
#
|
318
|
+
# @example
|
319
|
+
# require "google/cloud/bigquery"
|
320
|
+
#
|
321
|
+
# bigquery = Google::Cloud::Bigquery.new
|
322
|
+
#
|
323
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
324
|
+
# csv_table = bigquery.external csv_url
|
325
|
+
#
|
326
|
+
# csv_table.urls #=> ["gs://bucket/path/to/data.csv"]
|
327
|
+
#
|
328
|
+
def urls
|
329
|
+
@gapi.source_uris
|
330
|
+
end
|
331
|
+
|
332
|
+
##
|
333
|
+
# Indicates if the schema and format options are detected
|
334
|
+
# automatically.
|
335
|
+
#
|
336
|
+
# @return [Boolean]
|
337
|
+
#
|
338
|
+
# @example
|
339
|
+
# require "google/cloud/bigquery"
|
340
|
+
#
|
341
|
+
# bigquery = Google::Cloud::Bigquery.new
|
342
|
+
#
|
343
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
344
|
+
# csv_table = bigquery.external csv_url do |csv|
|
345
|
+
# csv.autodetect = true
|
346
|
+
# end
|
347
|
+
#
|
348
|
+
# csv_table.autodetect #=> true
|
349
|
+
#
|
350
|
+
def autodetect
|
351
|
+
@gapi.autodetect
|
352
|
+
end
|
353
|
+
|
354
|
+
##
|
355
|
+
# Set whether to detect schema and format options automatically. Any
|
356
|
+
# option specified explicitly will be honored.
|
357
|
+
#
|
358
|
+
# @param [Boolean] new_autodetect New autodetect value
|
359
|
+
#
|
360
|
+
# @example
|
361
|
+
# require "google/cloud/bigquery"
|
362
|
+
#
|
363
|
+
# bigquery = Google::Cloud::Bigquery.new
|
364
|
+
#
|
365
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
366
|
+
# csv_table = bigquery.external csv_url do |csv|
|
367
|
+
# csv.autodetect = true
|
368
|
+
# end
|
369
|
+
#
|
370
|
+
# csv_table.autodetect #=> true
|
371
|
+
#
|
372
|
+
def autodetect= new_autodetect
|
373
|
+
frozen_check!
|
374
|
+
@gapi.autodetect = new_autodetect
|
375
|
+
end
|
376
|
+
|
377
|
+
##
|
378
|
+
# The compression type of the data source. Possible values include
|
379
|
+
# `"GZIP"` and `nil`. The default value is `nil`. This setting is
|
380
|
+
# ignored for Google Cloud Bigtable, Google Cloud Datastore backups
|
381
|
+
# and Avro formats. Optional.
|
382
|
+
#
|
383
|
+
# @return [String]
|
384
|
+
#
|
385
|
+
# @example
|
386
|
+
# require "google/cloud/bigquery"
|
387
|
+
#
|
388
|
+
# bigquery = Google::Cloud::Bigquery.new
|
389
|
+
#
|
390
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
391
|
+
# csv_table = bigquery.external csv_url do |csv|
|
392
|
+
# csv.compression = "GZIP"
|
393
|
+
# end
|
394
|
+
#
|
395
|
+
# csv_table.compression #=> "GZIP"
|
396
|
+
def compression
|
397
|
+
@gapi.compression
|
398
|
+
end
|
399
|
+
|
400
|
+
##
|
401
|
+
# Set the compression type of the data source. Possible values include
|
402
|
+
# `"GZIP"` and `nil`. The default value is `nil`. This setting is
|
403
|
+
# ignored for Google Cloud Bigtable, Google Cloud Datastore backups
|
404
|
+
# and Avro formats. Optional.
|
405
|
+
#
|
406
|
+
# @param [String] new_compression New compression value
|
407
|
+
#
|
408
|
+
# @example
|
409
|
+
# require "google/cloud/bigquery"
|
410
|
+
#
|
411
|
+
# bigquery = Google::Cloud::Bigquery.new
|
412
|
+
#
|
413
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
414
|
+
# csv_table = bigquery.external csv_url do |csv|
|
415
|
+
# csv.compression = "GZIP"
|
416
|
+
# end
|
417
|
+
#
|
418
|
+
# csv_table.compression #=> "GZIP"
|
419
|
+
#
|
420
|
+
def compression= new_compression
|
421
|
+
frozen_check!
|
422
|
+
@gapi.compression = new_compression
|
423
|
+
end
|
424
|
+
|
425
|
+
##
|
426
|
+
# Indicates if BigQuery should allow extra values that are not
|
427
|
+
# represented in the table schema. If `true`, the extra values are
|
428
|
+
# ignored. If `false`, records with extra columns are treated as bad
|
429
|
+
# records, and if there are too many bad records, an invalid error is
|
430
|
+
# returned in the job result. The default value is `false`.
|
431
|
+
#
|
432
|
+
# BigQuery treats trailing columns as an extra in `CSV`, named values
|
433
|
+
# that don't match any column names in `JSON`. This setting is ignored
|
434
|
+
# for Google Cloud Bigtable, Google Cloud Datastore backups and Avro
|
435
|
+
# formats. Optional.
|
436
|
+
#
|
437
|
+
# @return [Boolean]
|
438
|
+
#
|
439
|
+
# @example
|
440
|
+
# require "google/cloud/bigquery"
|
441
|
+
#
|
442
|
+
# bigquery = Google::Cloud::Bigquery.new
|
443
|
+
#
|
444
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
445
|
+
# csv_table = bigquery.external csv_url do |csv|
|
446
|
+
# csv.ignore_unknown = true
|
447
|
+
# end
|
448
|
+
#
|
449
|
+
# csv_table.ignore_unknown #=> true
|
450
|
+
#
|
451
|
+
def ignore_unknown
|
452
|
+
@gapi.ignore_unknown_values
|
453
|
+
end
|
454
|
+
|
455
|
+
##
|
456
|
+
# Set whether BigQuery should allow extra values that are not
|
457
|
+
# represented in the table schema. If `true`, the extra values are
|
458
|
+
# ignored. If `false`, records with extra columns are treated as bad
|
459
|
+
# records, and if there are too many bad records, an invalid error is
|
460
|
+
# returned in the job result. The default value is `false`.
|
461
|
+
#
|
462
|
+
# BigQuery treats trailing columns as an extra in `CSV`, named values
|
463
|
+
# that don't match any column names in `JSON`. This setting is ignored
|
464
|
+
# for Google Cloud Bigtable, Google Cloud Datastore backups and Avro
|
465
|
+
# formats. Optional.
|
466
|
+
#
|
467
|
+
# @param [Boolean] new_ignore_unknown New ignore_unknown value
|
468
|
+
#
|
469
|
+
# @example
|
470
|
+
# require "google/cloud/bigquery"
|
471
|
+
#
|
472
|
+
# bigquery = Google::Cloud::Bigquery.new
|
473
|
+
#
|
474
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
475
|
+
# csv_table = bigquery.external csv_url do |csv|
|
476
|
+
# csv.ignore_unknown = true
|
477
|
+
# end
|
478
|
+
#
|
479
|
+
# csv_table.ignore_unknown #=> true
|
480
|
+
#
|
481
|
+
def ignore_unknown= new_ignore_unknown
|
482
|
+
frozen_check!
|
483
|
+
@gapi.ignore_unknown_values = new_ignore_unknown
|
484
|
+
end
|
485
|
+
|
486
|
+
##
|
487
|
+
# The maximum number of bad records that BigQuery can ignore when
|
488
|
+
# reading data. If the number of bad records exceeds this value, an
|
489
|
+
# invalid error is returned in the job result. The default value is 0,
|
490
|
+
# which requires that all records are valid. This setting is ignored
|
491
|
+
# for Google Cloud Bigtable, Google Cloud Datastore backups and Avro
|
492
|
+
# formats.
|
493
|
+
#
|
494
|
+
# @return [Integer]
|
495
|
+
#
|
496
|
+
# @example
|
497
|
+
# require "google/cloud/bigquery"
|
498
|
+
#
|
499
|
+
# bigquery = Google::Cloud::Bigquery.new
|
500
|
+
#
|
501
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
502
|
+
# csv_table = bigquery.external csv_url do |csv|
|
503
|
+
# csv.max_bad_records = 10
|
504
|
+
# end
|
505
|
+
#
|
506
|
+
# csv_table.max_bad_records #=> 10
|
507
|
+
#
|
508
|
+
def max_bad_records
|
509
|
+
@gapi.max_bad_records
|
510
|
+
end
|
511
|
+
|
512
|
+
##
|
513
|
+
# Set the maximum number of bad records that BigQuery can ignore when
|
514
|
+
# reading data. If the number of bad records exceeds this value, an
|
515
|
+
# invalid error is returned in the job result. The default value is 0,
|
516
|
+
# which requires that all records are valid. This setting is ignored
|
517
|
+
# for Google Cloud Bigtable, Google Cloud Datastore backups and Avro
|
518
|
+
# formats.
|
519
|
+
#
|
520
|
+
# @param [Integer] new_max_bad_records New max_bad_records value
|
521
|
+
#
|
522
|
+
# @example
|
523
|
+
# require "google/cloud/bigquery"
|
524
|
+
#
|
525
|
+
# bigquery = Google::Cloud::Bigquery.new
|
526
|
+
#
|
527
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
528
|
+
# csv_table = bigquery.external csv_url do |csv|
|
529
|
+
# csv.max_bad_records = 10
|
530
|
+
# end
|
531
|
+
#
|
532
|
+
# csv_table.max_bad_records #=> 10
|
533
|
+
#
|
534
|
+
def max_bad_records= new_max_bad_records
|
535
|
+
frozen_check!
|
536
|
+
@gapi.max_bad_records = new_max_bad_records
|
537
|
+
end
|
538
|
+
|
539
|
+
##
|
540
|
+
# @private Google API Client object.
|
541
|
+
def to_gapi
|
542
|
+
@gapi
|
543
|
+
end
|
544
|
+
|
545
|
+
##
|
546
|
+
# @private Google API Client object.
|
547
|
+
def self.from_gapi gapi
|
548
|
+
new_table = new
|
549
|
+
new_table.instance_variable_set :@gapi, gapi
|
550
|
+
new_table
|
551
|
+
end
|
552
|
+
|
553
|
+
protected
|
554
|
+
|
555
|
+
def frozen_check!
|
556
|
+
return unless frozen?
|
557
|
+
raise ArgumentError, "Cannot modify external data source when frozen"
|
558
|
+
end
|
559
|
+
end
|
560
|
+
|
561
|
+
##
|
562
|
+
# # CsvSource
|
563
|
+
#
|
564
|
+
# {External::CsvSource} is a subclass of {External::DataSource} and
|
565
|
+
# represents a CSV external data source that can be queried from
|
566
|
+
# directly, such as Google Cloud Storage or Google Drive, even though
|
567
|
+
# the data is not stored in BigQuery. Instead of loading or streaming
|
568
|
+
# the data, this object references the external data source.
|
569
|
+
#
|
570
|
+
# @example
|
571
|
+
# require "google/cloud/bigquery"
|
572
|
+
#
|
573
|
+
# bigquery = Google::Cloud::Bigquery.new
|
574
|
+
#
|
575
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
576
|
+
# csv_table = bigquery.external csv_url do |csv|
|
577
|
+
# csv.autodetect = true
|
578
|
+
# csv.skip_leading_rows = 1
|
579
|
+
# end
|
580
|
+
#
|
581
|
+
# data = bigquery.query "SELECT * FROM my_ext_table",
|
582
|
+
# external: { my_ext_table: csv_table }
|
583
|
+
#
|
584
|
+
# # Iterate over the first page of results
|
585
|
+
# data.each do |row|
|
586
|
+
# puts row[:name]
|
587
|
+
# end
|
588
|
+
# # Retrieve the next page of results
|
589
|
+
# data = data.next if data.next?
|
590
|
+
#
|
591
|
+
class CsvSource < External::DataSource
|
592
|
+
##
|
593
|
+
# @private Create an empty CsvSource object.
|
594
|
+
def initialize
|
595
|
+
super
|
596
|
+
@gapi.csv_options = Google::Apis::BigqueryV2::CsvOptions.new
|
597
|
+
end
|
598
|
+
|
599
|
+
##
|
600
|
+
# Indicates if BigQuery should accept rows that are missing trailing
|
601
|
+
# optional columns.
|
602
|
+
#
|
603
|
+
# @return [Boolean]
|
604
|
+
#
|
605
|
+
# @example
|
606
|
+
# require "google/cloud/bigquery"
|
607
|
+
#
|
608
|
+
# bigquery = Google::Cloud::Bigquery.new
|
609
|
+
#
|
610
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
611
|
+
# csv_table = bigquery.external csv_url do |csv|
|
612
|
+
# csv.jagged_rows = true
|
613
|
+
# end
|
614
|
+
#
|
615
|
+
# csv_table.jagged_rows #=> true
|
616
|
+
#
|
617
|
+
def jagged_rows
|
618
|
+
@gapi.csv_options.allow_jagged_rows
|
619
|
+
end
|
620
|
+
|
621
|
+
##
|
622
|
+
# Set whether BigQuery should accept rows that are missing trailing
|
623
|
+
# optional columns.
|
624
|
+
#
|
625
|
+
# @param [Boolean] new_jagged_rows New jagged_rows value
|
626
|
+
#
|
627
|
+
# @example
|
628
|
+
# require "google/cloud/bigquery"
|
629
|
+
#
|
630
|
+
# bigquery = Google::Cloud::Bigquery.new
|
631
|
+
#
|
632
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
633
|
+
# csv_table = bigquery.external csv_url do |csv|
|
634
|
+
# csv.jagged_rows = true
|
635
|
+
# end
|
636
|
+
#
|
637
|
+
# csv_table.jagged_rows #=> true
|
638
|
+
#
|
639
|
+
def jagged_rows= new_jagged_rows
|
640
|
+
frozen_check!
|
641
|
+
@gapi.csv_options.allow_jagged_rows = new_jagged_rows
|
642
|
+
end
|
643
|
+
|
644
|
+
##
|
645
|
+
# Indicates if BigQuery should allow quoted data sections that contain
|
646
|
+
# newline characters in a CSV file.
|
647
|
+
#
|
648
|
+
# @return [Boolean]
|
649
|
+
#
|
650
|
+
# @example
|
651
|
+
# require "google/cloud/bigquery"
|
652
|
+
#
|
653
|
+
# bigquery = Google::Cloud::Bigquery.new
|
654
|
+
#
|
655
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
656
|
+
# csv_table = bigquery.external csv_url do |csv|
|
657
|
+
# csv.quoted_newlines = true
|
658
|
+
# end
|
659
|
+
#
|
660
|
+
# csv_table.quoted_newlines #=> true
|
661
|
+
#
|
662
|
+
def quoted_newlines
|
663
|
+
@gapi.csv_options.allow_quoted_newlines
|
664
|
+
end
|
665
|
+
|
666
|
+
##
|
667
|
+
# Set whether BigQuery should allow quoted data sections that contain
|
668
|
+
# newline characters in a CSV file.
|
669
|
+
#
|
670
|
+
# @param [Boolean] new_quoted_newlines New quoted_newlines value
|
671
|
+
#
|
672
|
+
# @example
|
673
|
+
# require "google/cloud/bigquery"
|
674
|
+
#
|
675
|
+
# bigquery = Google::Cloud::Bigquery.new
|
676
|
+
#
|
677
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
678
|
+
# csv_table = bigquery.external csv_url do |csv|
|
679
|
+
# csv.quoted_newlines = true
|
680
|
+
# end
|
681
|
+
#
|
682
|
+
# csv_table.quoted_newlines #=> true
|
683
|
+
#
|
684
|
+
def quoted_newlines= new_quoted_newlines
|
685
|
+
frozen_check!
|
686
|
+
@gapi.csv_options.allow_quoted_newlines = new_quoted_newlines
|
687
|
+
end
|
688
|
+
|
689
|
+
##
|
690
|
+
# The character encoding of the data.
|
691
|
+
#
|
692
|
+
# @return [String]
|
693
|
+
#
|
694
|
+
# @example
|
695
|
+
# require "google/cloud/bigquery"
|
696
|
+
#
|
697
|
+
# bigquery = Google::Cloud::Bigquery.new
|
698
|
+
#
|
699
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
700
|
+
# csv_table = bigquery.external csv_url do |csv|
|
701
|
+
# csv.encoding = "UTF-8"
|
702
|
+
# end
|
703
|
+
#
|
704
|
+
# csv_table.encoding #=> "UTF-8"
|
705
|
+
#
|
706
|
+
def encoding
|
707
|
+
@gapi.csv_options.encoding
|
708
|
+
end
|
709
|
+
|
710
|
+
##
|
711
|
+
# Set the character encoding of the data.
|
712
|
+
#
|
713
|
+
# @param [String] new_encoding New encoding value
|
714
|
+
#
|
715
|
+
# @example
|
716
|
+
# require "google/cloud/bigquery"
|
717
|
+
#
|
718
|
+
# bigquery = Google::Cloud::Bigquery.new
|
719
|
+
#
|
720
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
721
|
+
# csv_table = bigquery.external csv_url do |csv|
|
722
|
+
# csv.encoding = "UTF-8"
|
723
|
+
# end
|
724
|
+
#
|
725
|
+
# csv_table.encoding #=> "UTF-8"
|
726
|
+
#
|
727
|
+
def encoding= new_encoding
|
728
|
+
frozen_check!
|
729
|
+
@gapi.csv_options.encoding = new_encoding
|
730
|
+
end
|
731
|
+
|
732
|
+
##
|
733
|
+
# Checks if the character encoding of the data is "UTF-8". This is the
|
734
|
+
# default.
|
735
|
+
#
|
736
|
+
# @return [Boolean]
|
737
|
+
#
|
738
|
+
# @example
|
739
|
+
# require "google/cloud/bigquery"
|
740
|
+
#
|
741
|
+
# bigquery = Google::Cloud::Bigquery.new
|
742
|
+
#
|
743
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
744
|
+
# csv_table = bigquery.external csv_url do |csv|
|
745
|
+
# csv.encoding = "UTF-8"
|
746
|
+
# end
|
747
|
+
#
|
748
|
+
# csv_table.encoding #=> "UTF-8"
|
749
|
+
# csv_table.utf8? #=> true
|
750
|
+
#
|
751
|
+
def utf8?
|
752
|
+
return true if encoding.nil?
|
753
|
+
encoding == "UTF-8"
|
754
|
+
end
|
755
|
+
|
756
|
+
##
|
757
|
+
# Checks if the character encoding of the data is "ISO-8859-1".
|
758
|
+
#
|
759
|
+
# @return [Boolean]
|
760
|
+
#
|
761
|
+
# @example
|
762
|
+
# require "google/cloud/bigquery"
|
763
|
+
#
|
764
|
+
# bigquery = Google::Cloud::Bigquery.new
|
765
|
+
#
|
766
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
767
|
+
# csv_table = bigquery.external csv_url do |csv|
|
768
|
+
# csv.encoding = "ISO-8859-1"
|
769
|
+
# end
|
770
|
+
#
|
771
|
+
# csv_table.encoding #=> "ISO-8859-1"
|
772
|
+
# csv_table.iso8859_1? #=> true
|
773
|
+
#
|
774
|
+
def iso8859_1?
|
775
|
+
encoding == "ISO-8859-1"
|
776
|
+
end
|
777
|
+
|
778
|
+
##
|
779
|
+
# The separator for fields in a CSV file.
|
780
|
+
#
|
781
|
+
# @return [String]
|
782
|
+
#
|
783
|
+
# @example
|
784
|
+
# require "google/cloud/bigquery"
|
785
|
+
#
|
786
|
+
# bigquery = Google::Cloud::Bigquery.new
|
787
|
+
#
|
788
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
789
|
+
# csv_table = bigquery.external csv_url do |csv|
|
790
|
+
# csv.delimiter = "|"
|
791
|
+
# end
|
792
|
+
#
|
793
|
+
# csv_table.delimiter #=> "|"
|
794
|
+
#
|
795
|
+
def delimiter
|
796
|
+
@gapi.csv_options.field_delimiter
|
797
|
+
end
|
798
|
+
|
799
|
+
##
|
800
|
+
# Set the separator for fields in a CSV file.
|
801
|
+
#
|
802
|
+
# @param [String] new_delimiter New delimiter value
|
803
|
+
#
|
804
|
+
# @example
|
805
|
+
# require "google/cloud/bigquery"
|
806
|
+
#
|
807
|
+
# bigquery = Google::Cloud::Bigquery.new
|
808
|
+
#
|
809
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
810
|
+
# csv_table = bigquery.external csv_url do |csv|
|
811
|
+
# csv.delimiter = "|"
|
812
|
+
# end
|
813
|
+
#
|
814
|
+
# csv_table.delimiter #=> "|"
|
815
|
+
#
|
816
|
+
def delimiter= new_delimiter
|
817
|
+
frozen_check!
|
818
|
+
@gapi.csv_options.field_delimiter = new_delimiter
|
819
|
+
end
|
820
|
+
|
821
|
+
##
|
822
|
+
# The value that is used to quote data sections in a CSV file.
|
823
|
+
#
|
824
|
+
# @return [String]
|
825
|
+
#
|
826
|
+
# @example
|
827
|
+
# require "google/cloud/bigquery"
|
828
|
+
#
|
829
|
+
# bigquery = Google::Cloud::Bigquery.new
|
830
|
+
#
|
831
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
832
|
+
# csv_table = bigquery.external csv_url do |csv|
|
833
|
+
# csv.quote = "'"
|
834
|
+
# end
|
835
|
+
#
|
836
|
+
# csv_table.quote #=> "'"
|
837
|
+
#
|
838
|
+
def quote
|
839
|
+
@gapi.csv_options.quote
|
840
|
+
end
|
841
|
+
|
842
|
+
##
|
843
|
+
# Set the value that is used to quote data sections in a CSV file.
|
844
|
+
#
|
845
|
+
# @param [String] new_quote New quote value
|
846
|
+
#
|
847
|
+
# @example
|
848
|
+
# require "google/cloud/bigquery"
|
849
|
+
#
|
850
|
+
# bigquery = Google::Cloud::Bigquery.new
|
851
|
+
#
|
852
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
853
|
+
# csv_table = bigquery.external csv_url do |csv|
|
854
|
+
# csv.quote = "'"
|
855
|
+
# end
|
856
|
+
#
|
857
|
+
# csv_table.quote #=> "'"
|
858
|
+
#
|
859
|
+
def quote= new_quote
|
860
|
+
frozen_check!
|
861
|
+
@gapi.csv_options.quote = new_quote
|
862
|
+
end
|
863
|
+
|
864
|
+
##
|
865
|
+
# The number of rows at the top of a CSV file that BigQuery will skip
|
866
|
+
# when reading the data.
|
867
|
+
#
|
868
|
+
# @return [Integer]
|
869
|
+
#
|
870
|
+
# @example
|
871
|
+
# require "google/cloud/bigquery"
|
872
|
+
#
|
873
|
+
# bigquery = Google::Cloud::Bigquery.new
|
874
|
+
#
|
875
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
876
|
+
# csv_table = bigquery.external csv_url do |csv|
|
877
|
+
# csv.skip_leading_rows = 1
|
878
|
+
# end
|
879
|
+
#
|
880
|
+
# csv_table.skip_leading_rows #=> 1
|
881
|
+
#
|
882
|
+
def skip_leading_rows
|
883
|
+
@gapi.csv_options.skip_leading_rows
|
884
|
+
end
|
885
|
+
|
886
|
+
##
|
887
|
+
# Set the number of rows at the top of a CSV file that BigQuery will
|
888
|
+
# skip when reading the data.
|
889
|
+
#
|
890
|
+
# @param [Integer] row_count New skip_leading_rows value
|
891
|
+
#
|
892
|
+
# @example
|
893
|
+
# require "google/cloud/bigquery"
|
894
|
+
#
|
895
|
+
# bigquery = Google::Cloud::Bigquery.new
|
896
|
+
#
|
897
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
898
|
+
# csv_table = bigquery.external csv_url do |csv|
|
899
|
+
# csv.skip_leading_rows = 1
|
900
|
+
# end
|
901
|
+
#
|
902
|
+
# csv_table.skip_leading_rows #=> 1
|
903
|
+
#
|
904
|
+
def skip_leading_rows= row_count
|
905
|
+
frozen_check!
|
906
|
+
@gapi.csv_options.skip_leading_rows = row_count
|
907
|
+
end
|
908
|
+
|
909
|
+
##
|
910
|
+
# The schema for the data.
|
911
|
+
#
|
912
|
+
# @param [Boolean] replace Whether to replace the existing schema with
|
913
|
+
# the new schema. If `true`, the fields will replace the existing
|
914
|
+
# schema. If `false`, the fields will be added to the existing
|
915
|
+
# schema. The default value is `false`.
|
916
|
+
# @yield [schema] a block for setting the schema
|
917
|
+
# @yieldparam [Schema] schema the object accepting the schema
|
918
|
+
#
|
919
|
+
# @return [Google::Cloud::Bigquery::Schema]
|
920
|
+
#
|
921
|
+
# @example
|
922
|
+
# require "google/cloud/bigquery"
|
923
|
+
#
|
924
|
+
# bigquery = Google::Cloud::Bigquery.new
|
925
|
+
#
|
926
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
927
|
+
# csv_table = bigquery.external csv_url do |csv|
|
928
|
+
# csv.schema do |schema|
|
929
|
+
# schema.string "name", mode: :required
|
930
|
+
# schema.string "email", mode: :required
|
931
|
+
# schema.integer "age", mode: :required
|
932
|
+
# schema.boolean "active", mode: :required
|
933
|
+
# end
|
934
|
+
# end
|
935
|
+
#
|
936
|
+
def schema replace: false
|
937
|
+
@schema ||= Schema.from_gapi @gapi.schema
|
938
|
+
if replace
|
939
|
+
frozen_check!
|
940
|
+
@schema = Schema.from_gapi
|
941
|
+
end
|
942
|
+
@schema.freeze if frozen?
|
943
|
+
yield @schema if block_given?
|
944
|
+
@schema
|
945
|
+
end
|
946
|
+
|
947
|
+
##
|
948
|
+
# Set the schema for the data.
|
949
|
+
#
|
950
|
+
# @param [Schema] new_schema The schema object.
|
951
|
+
#
|
952
|
+
# @example
|
953
|
+
# require "google/cloud/bigquery"
|
954
|
+
#
|
955
|
+
# bigquery = Google::Cloud::Bigquery.new
|
956
|
+
#
|
957
|
+
# csv_shema = bigquery.schema do |schema|
|
958
|
+
# schema.string "name", mode: :required
|
959
|
+
# schema.string "email", mode: :required
|
960
|
+
# schema.integer "age", mode: :required
|
961
|
+
# schema.boolean "active", mode: :required
|
962
|
+
# end
|
963
|
+
#
|
964
|
+
# csv_url = "gs://bucket/path/to/data.csv"
|
965
|
+
# csv_table = bigquery.external csv_url
|
966
|
+
# csv_table.schema = csv_shema
|
967
|
+
#
|
968
|
+
def schema= new_schema
|
969
|
+
frozen_check!
|
970
|
+
@schema = new_schema
|
971
|
+
end
|
972
|
+
|
973
|
+
##
|
974
|
+
# The fields of the schema.
|
975
|
+
#
|
976
|
+
# @return [Array<Schema::Field>] An array of field objects.
|
977
|
+
#
|
978
|
+
def fields
|
979
|
+
schema.fields
|
980
|
+
end
|
981
|
+
|
982
|
+
##
|
983
|
+
# The names of the columns in the schema.
|
984
|
+
#
|
985
|
+
# @return [Array<Symbol>] An array of column names.
|
986
|
+
#
|
987
|
+
def headers
|
988
|
+
schema.headers
|
989
|
+
end
|
990
|
+
|
991
|
+
##
|
992
|
+
# The types of the fields in the data in the schema, using the same
|
993
|
+
# format as the optional query parameter types.
|
994
|
+
#
|
995
|
+
# @return [Hash] A hash with field names as keys, and types as values.
|
996
|
+
#
|
997
|
+
def param_types
|
998
|
+
schema.param_types
|
999
|
+
end
|
1000
|
+
|
1001
|
+
##
|
1002
|
+
# @private Google API Client object.
|
1003
|
+
def to_gapi
|
1004
|
+
@gapi.schema = @schema.to_gapi if @schema
|
1005
|
+
@gapi
|
1006
|
+
end
|
1007
|
+
|
1008
|
+
##
|
1009
|
+
# @private Google API Client object.
|
1010
|
+
def self.from_gapi gapi
|
1011
|
+
new_table = super
|
1012
|
+
schema = Schema.from_gapi gapi.schema
|
1013
|
+
new_table.instance_variable_set :@schema, schema
|
1014
|
+
new_table
|
1015
|
+
end
|
1016
|
+
end
|
1017
|
+
|
1018
|
+
##
|
1019
|
+
# # JsonSource
|
1020
|
+
#
|
1021
|
+
# {External::JsonSource} is a subclass of {External::DataSource} and
|
1022
|
+
# represents a JSON external data source that can be queried from
|
1023
|
+
# directly, such as Google Cloud Storage or Google Drive, even though
|
1024
|
+
# the data is not stored in BigQuery. Instead of loading or streaming
|
1025
|
+
# the data, this object references the external data source.
|
1026
|
+
#
|
1027
|
+
# @example
|
1028
|
+
# require "google/cloud/bigquery"
|
1029
|
+
#
|
1030
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1031
|
+
#
|
1032
|
+
# require "google/cloud/bigquery"
|
1033
|
+
#
|
1034
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1035
|
+
#
|
1036
|
+
# json_url = "gs://bucket/path/to/data.json"
|
1037
|
+
# json_table = bigquery.external json_url do |json|
|
1038
|
+
# json.schema do |schema|
|
1039
|
+
# schema.string "name", mode: :required
|
1040
|
+
# schema.string "email", mode: :required
|
1041
|
+
# schema.integer "age", mode: :required
|
1042
|
+
# schema.boolean "active", mode: :required
|
1043
|
+
# end
|
1044
|
+
# end
|
1045
|
+
#
|
1046
|
+
# data = bigquery.query "SELECT * FROM my_ext_table",
|
1047
|
+
# external: { my_ext_table: json_table }
|
1048
|
+
#
|
1049
|
+
# # Iterate over the first page of results
|
1050
|
+
# data.each do |row|
|
1051
|
+
# puts row[:name]
|
1052
|
+
# end
|
1053
|
+
# # Retrieve the next page of results
|
1054
|
+
# data = data.next if data.next?
|
1055
|
+
#
|
1056
|
+
class JsonSource < External::DataSource
|
1057
|
+
##
|
1058
|
+
# The schema for the data.
|
1059
|
+
#
|
1060
|
+
# @param [Boolean] replace Whether to replace the existing schema with
|
1061
|
+
# the new schema. If `true`, the fields will replace the existing
|
1062
|
+
# schema. If `false`, the fields will be added to the existing
|
1063
|
+
# schema. The default value is `false`.
|
1064
|
+
# @yield [schema] a block for setting the schema
|
1065
|
+
# @yieldparam [Schema] schema the object accepting the schema
|
1066
|
+
#
|
1067
|
+
# @return [Google::Cloud::Bigquery::Schema]
|
1068
|
+
#
|
1069
|
+
# @example
|
1070
|
+
# require "google/cloud/bigquery"
|
1071
|
+
#
|
1072
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1073
|
+
#
|
1074
|
+
# json_url = "gs://bucket/path/to/data.json"
|
1075
|
+
# json_table = bigquery.external json_url do |json|
|
1076
|
+
# json.schema do |schema|
|
1077
|
+
# schema.string "name", mode: :required
|
1078
|
+
# schema.string "email", mode: :required
|
1079
|
+
# schema.integer "age", mode: :required
|
1080
|
+
# schema.boolean "active", mode: :required
|
1081
|
+
# end
|
1082
|
+
# end
|
1083
|
+
#
|
1084
|
+
def schema replace: false
|
1085
|
+
@schema ||= Schema.from_gapi @gapi.schema
|
1086
|
+
if replace
|
1087
|
+
frozen_check!
|
1088
|
+
@schema = Schema.from_gapi
|
1089
|
+
end
|
1090
|
+
@schema.freeze if frozen?
|
1091
|
+
yield @schema if block_given?
|
1092
|
+
@schema
|
1093
|
+
end
|
1094
|
+
|
1095
|
+
##
|
1096
|
+
# Set the schema for the data.
|
1097
|
+
#
|
1098
|
+
# @param [Schema] new_schema The schema object.
|
1099
|
+
#
|
1100
|
+
# @example
|
1101
|
+
# require "google/cloud/bigquery"
|
1102
|
+
#
|
1103
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1104
|
+
#
|
1105
|
+
# json_shema = bigquery.schema do |schema|
|
1106
|
+
# schema.string "name", mode: :required
|
1107
|
+
# schema.string "email", mode: :required
|
1108
|
+
# schema.integer "age", mode: :required
|
1109
|
+
# schema.boolean "active", mode: :required
|
1110
|
+
# end
|
1111
|
+
#
|
1112
|
+
# json_url = "gs://bucket/path/to/data.json"
|
1113
|
+
# json_table = bigquery.external json_url
|
1114
|
+
# json_table.schema = json_shema
|
1115
|
+
#
|
1116
|
+
def schema= new_schema
|
1117
|
+
frozen_check!
|
1118
|
+
@schema = new_schema
|
1119
|
+
end
|
1120
|
+
|
1121
|
+
##
|
1122
|
+
# The fields of the schema.
|
1123
|
+
#
|
1124
|
+
# @return [Array<Schema::Field>] An array of field objects.
|
1125
|
+
#
|
1126
|
+
def fields
|
1127
|
+
schema.fields
|
1128
|
+
end
|
1129
|
+
|
1130
|
+
##
|
1131
|
+
# The names of the columns in the schema.
|
1132
|
+
#
|
1133
|
+
# @return [Array<Symbol>] An array of column names.
|
1134
|
+
#
|
1135
|
+
def headers
|
1136
|
+
schema.headers
|
1137
|
+
end
|
1138
|
+
|
1139
|
+
##
|
1140
|
+
# The types of the fields in the data in the schema, using the same
|
1141
|
+
# format as the optional query parameter types.
|
1142
|
+
#
|
1143
|
+
# @return [Hash] A hash with field names as keys, and types as values.
|
1144
|
+
#
|
1145
|
+
def param_types
|
1146
|
+
schema.param_types
|
1147
|
+
end
|
1148
|
+
|
1149
|
+
##
|
1150
|
+
# @private Google API Client object.
|
1151
|
+
def to_gapi
|
1152
|
+
@gapi.schema = @schema.to_gapi if @schema
|
1153
|
+
@gapi
|
1154
|
+
end
|
1155
|
+
|
1156
|
+
##
|
1157
|
+
# @private Google API Client object.
|
1158
|
+
def self.from_gapi gapi
|
1159
|
+
new_table = super
|
1160
|
+
schema = Schema.from_gapi gapi.schema
|
1161
|
+
new_table.instance_variable_set :@schema, schema
|
1162
|
+
new_table
|
1163
|
+
end
|
1164
|
+
end
|
1165
|
+
|
1166
|
+
##
|
1167
|
+
# # SheetsSource
|
1168
|
+
#
|
1169
|
+
# {External::SheetsSource} is a subclass of {External::DataSource} and
|
1170
|
+
# represents a Google Sheets external data source that can be queried
|
1171
|
+
# from directly, even though the data is not stored in BigQuery. Instead
|
1172
|
+
# of loading or streaming the data, this object references the external
|
1173
|
+
# data source.
|
1174
|
+
#
|
1175
|
+
# @example
|
1176
|
+
# require "google/cloud/bigquery"
|
1177
|
+
#
|
1178
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1179
|
+
#
|
1180
|
+
# sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
|
1181
|
+
# sheets_table = bigquery.external sheets_url do |sheets|
|
1182
|
+
# sheets.skip_leading_rows = 1
|
1183
|
+
# end
|
1184
|
+
#
|
1185
|
+
# data = bigquery.query "SELECT * FROM my_ext_table",
|
1186
|
+
# external: { my_ext_table: sheets_table }
|
1187
|
+
#
|
1188
|
+
# # Iterate over the first page of results
|
1189
|
+
# data.each do |row|
|
1190
|
+
# puts row[:name]
|
1191
|
+
# end
|
1192
|
+
# # Retrieve the next page of results
|
1193
|
+
# data = data.next if data.next?
|
1194
|
+
#
|
1195
|
+
class SheetsSource < External::DataSource
|
1196
|
+
##
|
1197
|
+
# @private Create an empty SheetsSource object.
|
1198
|
+
def initialize
|
1199
|
+
super
|
1200
|
+
@gapi.google_sheets_options = Google::Apis::BigqueryV2::GoogleSheetsOptions.new
|
1201
|
+
end
|
1202
|
+
|
1203
|
+
##
|
1204
|
+
# The number of rows at the top of a sheet that BigQuery will skip
|
1205
|
+
# when reading the data. The default value is `0`.
|
1206
|
+
#
|
1207
|
+
# This property is useful if you have header rows that should be
|
1208
|
+
# skipped. When `autodetect` is on, behavior is the following:
|
1209
|
+
#
|
1210
|
+
# * `nil` - Autodetect tries to detect headers in the first row. If
|
1211
|
+
# they are not detected, the row is read as data. Otherwise data is
|
1212
|
+
# read starting from the second row.
|
1213
|
+
# * `0` - Instructs autodetect that there are no headers and data
|
1214
|
+
# should be read starting from the first row.
|
1215
|
+
# * `N > 0` - Autodetect skips `N-1` rows and tries to detect headers
|
1216
|
+
# in row `N`. If headers are not detected, row `N` is just skipped.
|
1217
|
+
# Otherwise row `N` is used to extract column names for the detected
|
1218
|
+
# schema.
|
1219
|
+
#
|
1220
|
+
# @return [Integer]
|
1221
|
+
#
|
1222
|
+
# @example
|
1223
|
+
# require "google/cloud/bigquery"
|
1224
|
+
#
|
1225
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1226
|
+
#
|
1227
|
+
# sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
|
1228
|
+
# sheets_table = bigquery.external sheets_url do |sheets|
|
1229
|
+
# sheets.skip_leading_rows = 1
|
1230
|
+
# end
|
1231
|
+
#
|
1232
|
+
# sheets_table.skip_leading_rows #=> 1
|
1233
|
+
#
|
1234
|
+
def skip_leading_rows
|
1235
|
+
@gapi.google_sheets_options.skip_leading_rows
|
1236
|
+
end
|
1237
|
+
|
1238
|
+
##
|
1239
|
+
# Set the number of rows at the top of a sheet that BigQuery will skip
|
1240
|
+
# when reading the data.
|
1241
|
+
#
|
1242
|
+
# @param [Integer] row_count New skip_leading_rows value
|
1243
|
+
#
|
1244
|
+
# @example
|
1245
|
+
# require "google/cloud/bigquery"
|
1246
|
+
#
|
1247
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1248
|
+
#
|
1249
|
+
# sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
|
1250
|
+
# sheets_table = bigquery.external sheets_url do |sheets|
|
1251
|
+
# sheets.skip_leading_rows = 1
|
1252
|
+
# end
|
1253
|
+
#
|
1254
|
+
# sheets_table.skip_leading_rows #=> 1
|
1255
|
+
#
|
1256
|
+
def skip_leading_rows= row_count
|
1257
|
+
frozen_check!
|
1258
|
+
@gapi.google_sheets_options.skip_leading_rows = row_count
|
1259
|
+
end
|
1260
|
+
|
1261
|
+
##
|
1262
|
+
# Range of a sheet to query from. Only used when non-empty. Typical
|
1263
|
+
# format: `{sheet_name}!{top_left_cell_id}:{bottom_right_cell_id}`.
|
1264
|
+
#
|
1265
|
+
# @return [String] Range of a sheet to query from.
|
1266
|
+
#
|
1267
|
+
# @example
|
1268
|
+
# require "google/cloud/bigquery"
|
1269
|
+
#
|
1270
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1271
|
+
#
|
1272
|
+
# sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
|
1273
|
+
# sheets_table = bigquery.external sheets_url do |sheets|
|
1274
|
+
# sheets.range = "sheet1!A1:B20"
|
1275
|
+
# end
|
1276
|
+
#
|
1277
|
+
# sheets_table.range #=> "sheet1!A1:B20"
|
1278
|
+
#
|
1279
|
+
def range
|
1280
|
+
@gapi.google_sheets_options.range
|
1281
|
+
end
|
1282
|
+
|
1283
|
+
##
|
1284
|
+
# Set the range of a sheet to query from. Only used when non-empty.
|
1285
|
+
# Typical format:
|
1286
|
+
# `{sheet_name}!{top_left_cell_id}:{bottom_right_cell_id}`.
|
1287
|
+
#
|
1288
|
+
# @param [String] new_range New range of a sheet to query from.
|
1289
|
+
#
|
1290
|
+
# @example
|
1291
|
+
# require "google/cloud/bigquery"
|
1292
|
+
#
|
1293
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1294
|
+
#
|
1295
|
+
# sheets_url = "https://docs.google.com/spreadsheets/d/1234567980"
|
1296
|
+
# sheets_table = bigquery.external sheets_url do |sheets|
|
1297
|
+
# sheets.range = "sheet1!A1:B20"
|
1298
|
+
# end
|
1299
|
+
#
|
1300
|
+
# sheets_table.range #=> "sheet1!A1:B20"
|
1301
|
+
#
|
1302
|
+
def range= new_range
|
1303
|
+
frozen_check!
|
1304
|
+
@gapi.google_sheets_options.range = new_range
|
1305
|
+
end
|
1306
|
+
end
|
1307
|
+
|
1308
|
+
##
|
1309
|
+
# # BigtableSource
|
1310
|
+
#
|
1311
|
+
# {External::BigtableSource} is a subclass of {External::DataSource} and
|
1312
|
+
# represents a Bigtable external data source that can be queried from
|
1313
|
+
# directly, even though the data is not stored in BigQuery. Instead of
|
1314
|
+
# loading or streaming the data, this object references the external
|
1315
|
+
# data source.
|
1316
|
+
#
|
1317
|
+
# @example
|
1318
|
+
# require "google/cloud/bigquery"
|
1319
|
+
#
|
1320
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1321
|
+
#
|
1322
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1323
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1324
|
+
# bt.rowkey_as_string = true
|
1325
|
+
# bt.add_family "user" do |u|
|
1326
|
+
# u.add_string "name"
|
1327
|
+
# u.add_string "email"
|
1328
|
+
# u.add_integer "age"
|
1329
|
+
# u.add_boolean "active"
|
1330
|
+
# end
|
1331
|
+
# end
|
1332
|
+
#
|
1333
|
+
# data = bigquery.query "SELECT * FROM my_ext_table",
|
1334
|
+
# external: { my_ext_table: bigtable_table }
|
1335
|
+
#
|
1336
|
+
# # Iterate over the first page of results
|
1337
|
+
# data.each do |row|
|
1338
|
+
# puts row[:name]
|
1339
|
+
# end
|
1340
|
+
# # Retrieve the next page of results
|
1341
|
+
# data = data.next if data.next?
|
1342
|
+
#
|
1343
|
+
class BigtableSource < External::DataSource
|
1344
|
+
##
|
1345
|
+
# @private Create an empty BigtableSource object.
|
1346
|
+
def initialize
|
1347
|
+
super
|
1348
|
+
@gapi.bigtable_options = Google::Apis::BigqueryV2::BigtableOptions.new
|
1349
|
+
@families = []
|
1350
|
+
end
|
1351
|
+
|
1352
|
+
##
|
1353
|
+
# List of column families to expose in the table schema along with
|
1354
|
+
# their types. This list restricts the column families that can be
|
1355
|
+
# referenced in queries and specifies their value types. You can use
|
1356
|
+
# this list to do type conversions - see
|
1357
|
+
# {BigtableSource::ColumnFamily#type} for more details. If you leave
|
1358
|
+
# this list empty, all column families are present in the table schema
|
1359
|
+
# and their values are read as `BYTES`. During a query only the column
|
1360
|
+
# families referenced in that query are read from Bigtable.
|
1361
|
+
#
|
1362
|
+
# @return [Array<BigtableSource::ColumnFamily>]
|
1363
|
+
#
|
1364
|
+
# @example
|
1365
|
+
# require "google/cloud/bigquery"
|
1366
|
+
#
|
1367
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1368
|
+
#
|
1369
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1370
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1371
|
+
# bt.rowkey_as_string = true
|
1372
|
+
# bt.add_family "user" do |u|
|
1373
|
+
# u.add_string "name"
|
1374
|
+
# u.add_string "email"
|
1375
|
+
# u.add_integer "age"
|
1376
|
+
# u.add_boolean "active"
|
1377
|
+
# end
|
1378
|
+
# end
|
1379
|
+
#
|
1380
|
+
# bigtable_table.families.count #=> 1
|
1381
|
+
#
|
1382
|
+
def families
|
1383
|
+
@families
|
1384
|
+
end
|
1385
|
+
|
1386
|
+
##
|
1387
|
+
# Add a column family to expose in the table schema along with its
|
1388
|
+
# types. Columns belonging to the column family may also be exposed.
|
1389
|
+
#
|
1390
|
+
# @param [String] family_id Identifier of the column family. See
|
1391
|
+
# {BigtableSource::ColumnFamily#family_id}.
|
1392
|
+
# @param [String] encoding The encoding of the values when the type is
|
1393
|
+
# not `STRING`. See {BigtableSource::ColumnFamily#encoding}.
|
1394
|
+
# @param [Boolean] latest Whether only the latest version of value are
|
1395
|
+
# exposed for all columns in this column family. See
|
1396
|
+
# {BigtableSource::ColumnFamily#latest}.
|
1397
|
+
# @param [String] type The type to convert the value in cells of this
|
1398
|
+
# column. See {BigtableSource::ColumnFamily#type}.
|
1399
|
+
#
|
1400
|
+
# @yield [family] a block for setting the family
|
1401
|
+
# @yieldparam [BigtableSource::ColumnFamily] family the family object
|
1402
|
+
#
|
1403
|
+
# @return [BigtableSource::ColumnFamily]
|
1404
|
+
#
|
1405
|
+
# @example
|
1406
|
+
# require "google/cloud/bigquery"
|
1407
|
+
#
|
1408
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1409
|
+
#
|
1410
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1411
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1412
|
+
# bt.rowkey_as_string = true
|
1413
|
+
# bt.add_family "user" do |u|
|
1414
|
+
# u.add_string "name"
|
1415
|
+
# u.add_string "email"
|
1416
|
+
# u.add_integer "age"
|
1417
|
+
# u.add_boolean "active"
|
1418
|
+
# end
|
1419
|
+
# end
|
1420
|
+
#
|
1421
|
+
def add_family family_id, encoding: nil, latest: nil, type: nil
|
1422
|
+
frozen_check!
|
1423
|
+
fam = BigtableSource::ColumnFamily.new
|
1424
|
+
fam.family_id = family_id
|
1425
|
+
fam.encoding = encoding if encoding
|
1426
|
+
fam.latest = latest if latest
|
1427
|
+
fam.type = type if type
|
1428
|
+
yield fam if block_given?
|
1429
|
+
@families << fam
|
1430
|
+
fam
|
1431
|
+
end
|
1432
|
+
|
1433
|
+
##
|
1434
|
+
# Whether the rowkey column families will be read and converted to
|
1435
|
+
# string. Otherwise they are read with `BYTES` type values and users
|
1436
|
+
# need to manually cast them with `CAST` if necessary. The default
|
1437
|
+
# value is `false`.
|
1438
|
+
#
|
1439
|
+
# @return [Boolean]
|
1440
|
+
#
|
1441
|
+
# @example
|
1442
|
+
# require "google/cloud/bigquery"
|
1443
|
+
#
|
1444
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1445
|
+
#
|
1446
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1447
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1448
|
+
# bt.rowkey_as_string = true
|
1449
|
+
# end
|
1450
|
+
#
|
1451
|
+
# bigtable_table.rowkey_as_string #=> true
|
1452
|
+
#
|
1453
|
+
def rowkey_as_string
|
1454
|
+
@gapi.bigtable_options.read_rowkey_as_string
|
1455
|
+
end
|
1456
|
+
|
1457
|
+
##
|
1458
|
+
# Set the number of rows at the top of a sheet that BigQuery will skip
|
1459
|
+
# when reading the data.
|
1460
|
+
#
|
1461
|
+
# @param [Boolean] row_rowkey New rowkey_as_string value
|
1462
|
+
#
|
1463
|
+
# @example
|
1464
|
+
# require "google/cloud/bigquery"
|
1465
|
+
#
|
1466
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1467
|
+
#
|
1468
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1469
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1470
|
+
# bt.rowkey_as_string = true
|
1471
|
+
# end
|
1472
|
+
#
|
1473
|
+
# bigtable_table.rowkey_as_string #=> true
|
1474
|
+
#
|
1475
|
+
def rowkey_as_string= row_rowkey
|
1476
|
+
frozen_check!
|
1477
|
+
@gapi.bigtable_options.read_rowkey_as_string = row_rowkey
|
1478
|
+
end
|
1479
|
+
|
1480
|
+
##
|
1481
|
+
# @private Google API Client object.
|
1482
|
+
def to_gapi
|
1483
|
+
@gapi.bigtable_options.column_families = @families.map(&:to_gapi)
|
1484
|
+
@gapi
|
1485
|
+
end
|
1486
|
+
|
1487
|
+
##
|
1488
|
+
# @private Google API Client object.
|
1489
|
+
def self.from_gapi gapi
|
1490
|
+
new_table = super
|
1491
|
+
families = Array gapi.bigtable_options.column_families
|
1492
|
+
families = families.map { |fam_gapi| BigtableSource::ColumnFamily.from_gapi fam_gapi }
|
1493
|
+
new_table.instance_variable_set :@families, families
|
1494
|
+
new_table
|
1495
|
+
end
|
1496
|
+
|
1497
|
+
##
|
1498
|
+
# @private
|
1499
|
+
def freeze
|
1500
|
+
@families.map(&:freeze!)
|
1501
|
+
@families.freeze!
|
1502
|
+
super
|
1503
|
+
end
|
1504
|
+
|
1505
|
+
protected
|
1506
|
+
|
1507
|
+
def frozen_check!
|
1508
|
+
return unless frozen?
|
1509
|
+
raise ArgumentError, "Cannot modify external data source when frozen"
|
1510
|
+
end
|
1511
|
+
|
1512
|
+
##
|
1513
|
+
# # BigtableSource::ColumnFamily
|
1514
|
+
#
|
1515
|
+
# A Bigtable column family used to expose in the table schema along
|
1516
|
+
# with its types and columns.
|
1517
|
+
#
|
1518
|
+
# @example
|
1519
|
+
# require "google/cloud/bigquery"
|
1520
|
+
#
|
1521
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1522
|
+
#
|
1523
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1524
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1525
|
+
# bt.rowkey_as_string = true
|
1526
|
+
# bt.add_family "user" do |u|
|
1527
|
+
# u.add_string "name"
|
1528
|
+
# u.add_string "email"
|
1529
|
+
# u.add_integer "age"
|
1530
|
+
# u.add_boolean "active"
|
1531
|
+
# end
|
1532
|
+
# end
|
1533
|
+
#
|
1534
|
+
# data = bigquery.query "SELECT * FROM my_ext_table",
|
1535
|
+
# external: { my_ext_table: bigtable_table }
|
1536
|
+
#
|
1537
|
+
# # Iterate over the first page of results
|
1538
|
+
# data.each do |row|
|
1539
|
+
# puts row[:name]
|
1540
|
+
# end
|
1541
|
+
# # Retrieve the next page of results
|
1542
|
+
# data = data.next if data.next?
|
1543
|
+
#
|
1544
|
+
class ColumnFamily
|
1545
|
+
##
|
1546
|
+
# @private Create an empty BigtableSource::ColumnFamily object.
|
1547
|
+
def initialize
|
1548
|
+
@gapi = Google::Apis::BigqueryV2::BigtableColumnFamily.new
|
1549
|
+
@columns = []
|
1550
|
+
end
|
1551
|
+
|
1552
|
+
##
|
1553
|
+
# The encoding of the values when the type is not `STRING`.
|
1554
|
+
#
|
1555
|
+
# @return [String]
|
1556
|
+
#
|
1557
|
+
# @example
|
1558
|
+
# require "google/cloud/bigquery"
|
1559
|
+
#
|
1560
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1561
|
+
#
|
1562
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1563
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1564
|
+
# bt.add_family "user" do |u|
|
1565
|
+
# u.encoding = "UTF-8"
|
1566
|
+
# end
|
1567
|
+
# end
|
1568
|
+
#
|
1569
|
+
# bigtable_table.families[0].encoding #=> "UTF-8"
|
1570
|
+
#
|
1571
|
+
def encoding
|
1572
|
+
@gapi.encoding
|
1573
|
+
end
|
1574
|
+
|
1575
|
+
##
|
1576
|
+
# Set the encoding of the values when the type is not `STRING`.
|
1577
|
+
# Acceptable encoding values are:
|
1578
|
+
#
|
1579
|
+
# * `TEXT` - indicates values are alphanumeric text strings.
|
1580
|
+
# * `BINARY` - indicates values are encoded using HBase
|
1581
|
+
# `Bytes.toBytes` family of functions. This can be overridden on a
|
1582
|
+
# column.
|
1583
|
+
#
|
1584
|
+
# @param [String] new_encoding New encoding value
|
1585
|
+
#
|
1586
|
+
# @example
|
1587
|
+
# require "google/cloud/bigquery"
|
1588
|
+
#
|
1589
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1590
|
+
#
|
1591
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1592
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1593
|
+
# bt.add_family "user" do |u|
|
1594
|
+
# u.encoding = "UTF-8"
|
1595
|
+
# end
|
1596
|
+
# end
|
1597
|
+
#
|
1598
|
+
# bigtable_table.families[0].encoding #=> "UTF-8"
|
1599
|
+
#
|
1600
|
+
def encoding= new_encoding
|
1601
|
+
frozen_check!
|
1602
|
+
@gapi.encoding = new_encoding
|
1603
|
+
end
|
1604
|
+
|
1605
|
+
##
|
1606
|
+
# Identifier of the column family.
|
1607
|
+
#
|
1608
|
+
# @return [String]
|
1609
|
+
#
|
1610
|
+
# @example
|
1611
|
+
# require "google/cloud/bigquery"
|
1612
|
+
#
|
1613
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1614
|
+
#
|
1615
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1616
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1617
|
+
# bt.add_family "user"
|
1618
|
+
# end
|
1619
|
+
#
|
1620
|
+
# bigtable_table.families[0].family_id #=> "user"
|
1621
|
+
#
|
1622
|
+
def family_id
|
1623
|
+
@gapi.family_id
|
1624
|
+
end
|
1625
|
+
|
1626
|
+
##
|
1627
|
+
# Set the identifier of the column family.
|
1628
|
+
#
|
1629
|
+
# @param [String] new_family_id New family_id value
|
1630
|
+
#
|
1631
|
+
# @example
|
1632
|
+
# require "google/cloud/bigquery"
|
1633
|
+
#
|
1634
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1635
|
+
#
|
1636
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1637
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1638
|
+
# bt.add_family "user"
|
1639
|
+
# end
|
1640
|
+
#
|
1641
|
+
# bigtable_table.families[0].family_id #=> "user"
|
1642
|
+
# bigtable_table.families[0].family_id = "User"
|
1643
|
+
# bigtable_table.families[0].family_id #=> "User"
|
1644
|
+
#
|
1645
|
+
def family_id= new_family_id
|
1646
|
+
frozen_check!
|
1647
|
+
@gapi.family_id = new_family_id
|
1648
|
+
end
|
1649
|
+
|
1650
|
+
##
|
1651
|
+
# Whether only the latest version of value are exposed for all
|
1652
|
+
# columns in this column family.
|
1653
|
+
#
|
1654
|
+
# @return [Boolean]
|
1655
|
+
#
|
1656
|
+
# @example
|
1657
|
+
# require "google/cloud/bigquery"
|
1658
|
+
#
|
1659
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1660
|
+
#
|
1661
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1662
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1663
|
+
# bt.add_family "user" do |u|
|
1664
|
+
# u.latest = true
|
1665
|
+
# end
|
1666
|
+
# end
|
1667
|
+
#
|
1668
|
+
# bigtable_table.families[0].latest #=> true
|
1669
|
+
#
|
1670
|
+
def latest
|
1671
|
+
@gapi.only_read_latest
|
1672
|
+
end
|
1673
|
+
|
1674
|
+
##
|
1675
|
+
# Set whether only the latest version of value are exposed for all
|
1676
|
+
# columns in this column family.
|
1677
|
+
#
|
1678
|
+
# @param [Boolean] new_latest New latest value
|
1679
|
+
#
|
1680
|
+
# @example
|
1681
|
+
# require "google/cloud/bigquery"
|
1682
|
+
#
|
1683
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1684
|
+
#
|
1685
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1686
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1687
|
+
# bt.add_family "user" do |u|
|
1688
|
+
# u.latest = true
|
1689
|
+
# end
|
1690
|
+
# end
|
1691
|
+
#
|
1692
|
+
# bigtable_table.families[0].latest #=> true
|
1693
|
+
#
|
1694
|
+
def latest= new_latest
|
1695
|
+
frozen_check!
|
1696
|
+
@gapi.only_read_latest = new_latest
|
1697
|
+
end
|
1698
|
+
|
1699
|
+
##
|
1700
|
+
# The type to convert the value in cells of this column family. The
|
1701
|
+
# values are expected to be encoded using HBase `Bytes.toBytes`
|
1702
|
+
# function when using the `BINARY` encoding value. The following
|
1703
|
+
# BigQuery types are allowed:
|
1704
|
+
#
|
1705
|
+
# * `BYTES`
|
1706
|
+
# * `STRING`
|
1707
|
+
# * `INTEGER`
|
1708
|
+
# * `FLOAT`
|
1709
|
+
# * `BOOLEAN`
|
1710
|
+
#
|
1711
|
+
# Default type is `BYTES`. This can be overridden on a column.
|
1712
|
+
#
|
1713
|
+
# @return [String]
|
1714
|
+
#
|
1715
|
+
# @example
|
1716
|
+
# require "google/cloud/bigquery"
|
1717
|
+
#
|
1718
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1719
|
+
#
|
1720
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1721
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1722
|
+
# bt.add_family "user" do |u|
|
1723
|
+
# u.type = "STRING"
|
1724
|
+
# end
|
1725
|
+
# end
|
1726
|
+
#
|
1727
|
+
# bigtable_table.families[0].type #=> "STRING"
|
1728
|
+
#
|
1729
|
+
def type
|
1730
|
+
@gapi.type
|
1731
|
+
end
|
1732
|
+
|
1733
|
+
##
|
1734
|
+
# Set the type to convert the value in cells of this column family.
|
1735
|
+
# The values are expected to be encoded using HBase `Bytes.toBytes`
|
1736
|
+
# function when using the `BINARY` encoding value. The following
|
1737
|
+
# BigQuery types are allowed:
|
1738
|
+
#
|
1739
|
+
# * `BYTES`
|
1740
|
+
# * `STRING`
|
1741
|
+
# * `INTEGER`
|
1742
|
+
# * `FLOAT`
|
1743
|
+
# * `BOOLEAN`
|
1744
|
+
#
|
1745
|
+
# Default type is `BYTES`. This can be overridden on a column.
|
1746
|
+
#
|
1747
|
+
# @param [String] new_type New type value
|
1748
|
+
#
|
1749
|
+
# @example
|
1750
|
+
# require "google/cloud/bigquery"
|
1751
|
+
#
|
1752
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1753
|
+
#
|
1754
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1755
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1756
|
+
# bt.add_family "user" do |u|
|
1757
|
+
# u.type = "STRING"
|
1758
|
+
# end
|
1759
|
+
# end
|
1760
|
+
#
|
1761
|
+
# bigtable_table.families[0].type #=> "STRING"
|
1762
|
+
#
|
1763
|
+
def type= new_type
|
1764
|
+
frozen_check!
|
1765
|
+
@gapi.type = new_type
|
1766
|
+
end
|
1767
|
+
|
1768
|
+
##
|
1769
|
+
# Lists of columns that should be exposed as individual fields.
|
1770
|
+
#
|
1771
|
+
# @return [Array<BigtableSource::Column>]
|
1772
|
+
#
|
1773
|
+
# @example
|
1774
|
+
# require "google/cloud/bigquery"
|
1775
|
+
#
|
1776
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1777
|
+
#
|
1778
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1779
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1780
|
+
# bt.rowkey_as_string = true
|
1781
|
+
# bt.add_family "user" do |u|
|
1782
|
+
# u.add_string "name"
|
1783
|
+
# u.add_string "email"
|
1784
|
+
# u.add_integer "age"
|
1785
|
+
# u.add_boolean "active"
|
1786
|
+
# end
|
1787
|
+
# end
|
1788
|
+
#
|
1789
|
+
# bigtable_table.families[0].columns.count #=> 4
|
1790
|
+
#
|
1791
|
+
def columns
|
1792
|
+
@columns
|
1793
|
+
end
|
1794
|
+
|
1795
|
+
##
|
1796
|
+
# Add a column to the column family to expose in the table schema
|
1797
|
+
# along with its types.
|
1798
|
+
#
|
1799
|
+
# @param [String] qualifier Qualifier of the column. See
|
1800
|
+
# {BigtableSource::Column#qualifier}.
|
1801
|
+
# @param [String] as A valid identifier to be used as the column
|
1802
|
+
# field name if the qualifier is not a valid BigQuery field
|
1803
|
+
# identifier (i.e. does not match `[a-zA-Z][a-zA-Z0-9_]*`). See
|
1804
|
+
# {BigtableSource::Column#field_name}.
|
1805
|
+
# @param [String] type The type to convert the value in cells of
|
1806
|
+
# this column. See {BigtableSource::Column#type}. The following
|
1807
|
+
# BigQuery types are allowed:
|
1808
|
+
#
|
1809
|
+
# * `BYTES`
|
1810
|
+
# * `STRING`
|
1811
|
+
# * `INTEGER`
|
1812
|
+
# * `FLOAT`
|
1813
|
+
# * `BOOLEAN`
|
1814
|
+
#
|
1815
|
+
# @yield [column] a block for setting the column
|
1816
|
+
# @yieldparam [BigtableSource::Column] column the column object
|
1817
|
+
#
|
1818
|
+
# @return [Array<BigtableSource::Column>]
|
1819
|
+
#
|
1820
|
+
# @example
|
1821
|
+
# require "google/cloud/bigquery"
|
1822
|
+
#
|
1823
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1824
|
+
#
|
1825
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1826
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1827
|
+
# bt.rowkey_as_string = true
|
1828
|
+
# bt.add_family "user" do |u|
|
1829
|
+
# u.add_column "name", type: "STRING"
|
1830
|
+
# end
|
1831
|
+
# end
|
1832
|
+
#
|
1833
|
+
def add_column qualifier, as: nil, type: nil
|
1834
|
+
frozen_check!
|
1835
|
+
col = BigtableSource::Column.new
|
1836
|
+
col.qualifier = qualifier
|
1837
|
+
col.field_name = as if as
|
1838
|
+
col.type = type if type
|
1839
|
+
yield col if block_given?
|
1840
|
+
@columns << col
|
1841
|
+
col
|
1842
|
+
end
|
1843
|
+
|
1844
|
+
##
|
1845
|
+
# Add a column to the column family to expose in the table schema
|
1846
|
+
# that is specified as the `BYTES` type.
|
1847
|
+
#
|
1848
|
+
# @param [String] qualifier Qualifier of the column. See
|
1849
|
+
# {BigtableSource::Column#qualifier}.
|
1850
|
+
# @param [String] as A valid identifier to be used as the column
|
1851
|
+
# field name if the qualifier is not a valid BigQuery field
|
1852
|
+
# identifier (i.e. does not match `[a-zA-Z][a-zA-Z0-9_]*`). See
|
1853
|
+
# {BigtableSource::Column#field_name}.
|
1854
|
+
#
|
1855
|
+
# @yield [column] a block for setting the column
|
1856
|
+
# @yieldparam [BigtableSource::Column] column the column object
|
1857
|
+
#
|
1858
|
+
# @return [Array<BigtableSource::Column>]
|
1859
|
+
#
|
1860
|
+
# @example
|
1861
|
+
# require "google/cloud/bigquery"
|
1862
|
+
#
|
1863
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1864
|
+
#
|
1865
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1866
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1867
|
+
# bt.rowkey_as_string = true
|
1868
|
+
# bt.add_family "user" do |u|
|
1869
|
+
# u.add_bytes "avatar"
|
1870
|
+
# end
|
1871
|
+
# end
|
1872
|
+
#
|
1873
|
+
def add_bytes qualifier, as: nil
|
1874
|
+
col = add_column qualifier, as: as, type: "BYTES"
|
1875
|
+
yield col if block_given?
|
1876
|
+
col
|
1877
|
+
end
|
1878
|
+
|
1879
|
+
##
|
1880
|
+
# Add a column to the column family to expose in the table schema
|
1881
|
+
# that is specified as the `STRING` type.
|
1882
|
+
#
|
1883
|
+
# @param [String] qualifier Qualifier of the column. See
|
1884
|
+
# {BigtableSource::Column#qualifier}.
|
1885
|
+
# @param [String] as A valid identifier to be used as the column
|
1886
|
+
# field name if the qualifier is not a valid BigQuery field
|
1887
|
+
# identifier (i.e. does not match `[a-zA-Z][a-zA-Z0-9_]*`). See
|
1888
|
+
# {BigtableSource::Column#field_name}.
|
1889
|
+
#
|
1890
|
+
# @yield [column] a block for setting the column
|
1891
|
+
# @yieldparam [BigtableSource::Column] column the column object
|
1892
|
+
#
|
1893
|
+
# @return [Array<BigtableSource::Column>]
|
1894
|
+
#
|
1895
|
+
# @example
|
1896
|
+
# require "google/cloud/bigquery"
|
1897
|
+
#
|
1898
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1899
|
+
#
|
1900
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1901
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1902
|
+
# bt.rowkey_as_string = true
|
1903
|
+
# bt.add_family "user" do |u|
|
1904
|
+
# u.add_string "name"
|
1905
|
+
# end
|
1906
|
+
# end
|
1907
|
+
#
|
1908
|
+
def add_string qualifier, as: nil
|
1909
|
+
col = add_column qualifier, as: as, type: "STRING"
|
1910
|
+
yield col if block_given?
|
1911
|
+
col
|
1912
|
+
end
|
1913
|
+
|
1914
|
+
##
|
1915
|
+
# Add a column to the column family to expose in the table schema
|
1916
|
+
# that is specified as the `INTEGER` type.
|
1917
|
+
#
|
1918
|
+
# @param [String] qualifier Qualifier of the column. See
|
1919
|
+
# {BigtableSource::Column#qualifier}.
|
1920
|
+
# @param [String] as A valid identifier to be used as the column
|
1921
|
+
# field name if the qualifier is not a valid BigQuery field
|
1922
|
+
# identifier (i.e. does not match `[a-zA-Z][a-zA-Z0-9_]*`). See
|
1923
|
+
# {BigtableSource::Column#field_name}.
|
1924
|
+
#
|
1925
|
+
# @yield [column] a block for setting the column
|
1926
|
+
# @yieldparam [BigtableSource::Column] column the column object
|
1927
|
+
#
|
1928
|
+
# @return [Array<BigtableSource::Column>]
|
1929
|
+
#
|
1930
|
+
# @example
|
1931
|
+
# require "google/cloud/bigquery"
|
1932
|
+
#
|
1933
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1934
|
+
#
|
1935
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1936
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1937
|
+
# bt.rowkey_as_string = true
|
1938
|
+
# bt.add_family "user" do |u|
|
1939
|
+
# u.add_integer "age"
|
1940
|
+
# end
|
1941
|
+
# end
|
1942
|
+
#
|
1943
|
+
def add_integer qualifier, as: nil
|
1944
|
+
col = add_column qualifier, as: as, type: "INTEGER"
|
1945
|
+
yield col if block_given?
|
1946
|
+
col
|
1947
|
+
end
|
1948
|
+
|
1949
|
+
##
|
1950
|
+
# Add a column to the column family to expose in the table schema
|
1951
|
+
# that is specified as the `FLOAT` type.
|
1952
|
+
#
|
1953
|
+
# @param [String] qualifier Qualifier of the column. See
|
1954
|
+
# {BigtableSource::Column#qualifier}.
|
1955
|
+
# @param [String] as A valid identifier to be used as the column
|
1956
|
+
# field name if the qualifier is not a valid BigQuery field
|
1957
|
+
# identifier (i.e. does not match `[a-zA-Z][a-zA-Z0-9_]*`). See
|
1958
|
+
# {BigtableSource::Column#field_name}.
|
1959
|
+
#
|
1960
|
+
# @yield [column] a block for setting the column
|
1961
|
+
# @yieldparam [BigtableSource::Column] column the column object
|
1962
|
+
#
|
1963
|
+
# @return [Array<BigtableSource::Column>]
|
1964
|
+
#
|
1965
|
+
# @example
|
1966
|
+
# require "google/cloud/bigquery"
|
1967
|
+
#
|
1968
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1969
|
+
#
|
1970
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
1971
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
1972
|
+
# bt.rowkey_as_string = true
|
1973
|
+
# bt.add_family "user" do |u|
|
1974
|
+
# u.add_float "score"
|
1975
|
+
# end
|
1976
|
+
# end
|
1977
|
+
#
|
1978
|
+
def add_float qualifier, as: nil
|
1979
|
+
col = add_column qualifier, as: as, type: "FLOAT"
|
1980
|
+
yield col if block_given?
|
1981
|
+
col
|
1982
|
+
end
|
1983
|
+
|
1984
|
+
##
|
1985
|
+
# Add a column to the column family to expose in the table schema
|
1986
|
+
# that is specified as the `BOOLEAN` type.
|
1987
|
+
#
|
1988
|
+
# @param [String] qualifier Qualifier of the column. See
|
1989
|
+
# {BigtableSource::Column#qualifier}.
|
1990
|
+
# @param [String] as A valid identifier to be used as the column
|
1991
|
+
# field name if the qualifier is not a valid BigQuery field
|
1992
|
+
# identifier (i.e. does not match `[a-zA-Z][a-zA-Z0-9_]*`). See
|
1993
|
+
# {BigtableSource::Column#field_name}.
|
1994
|
+
#
|
1995
|
+
# @yield [column] a block for setting the column
|
1996
|
+
# @yieldparam [BigtableSource::Column] column the column object
|
1997
|
+
#
|
1998
|
+
# @return [Array<BigtableSource::Column>]
|
1999
|
+
#
|
2000
|
+
# @example
|
2001
|
+
# require "google/cloud/bigquery"
|
2002
|
+
#
|
2003
|
+
# bigquery = Google::Cloud::Bigquery.new
|
2004
|
+
#
|
2005
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
2006
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
2007
|
+
# bt.rowkey_as_string = true
|
2008
|
+
# bt.add_family "user" do |u|
|
2009
|
+
# u.add_boolean "active"
|
2010
|
+
# end
|
2011
|
+
# end
|
2012
|
+
#
|
2013
|
+
def add_boolean qualifier, as: nil
|
2014
|
+
col = add_column qualifier, as: as, type: "BOOLEAN"
|
2015
|
+
yield col if block_given?
|
2016
|
+
col
|
2017
|
+
end
|
2018
|
+
|
2019
|
+
##
|
2020
|
+
# @private Google API Client object.
|
2021
|
+
def to_gapi
|
2022
|
+
@gapi.columns = @columns.map(&:to_gapi)
|
2023
|
+
@gapi
|
2024
|
+
end
|
2025
|
+
|
2026
|
+
##
|
2027
|
+
# @private Google API Client object.
|
2028
|
+
def self.from_gapi gapi
|
2029
|
+
new_fam = new
|
2030
|
+
new_fam.instance_variable_set :@gapi, gapi
|
2031
|
+
columns = Array(gapi.columns).map { |col_gapi| BigtableSource::Column.from_gapi col_gapi }
|
2032
|
+
new_fam.instance_variable_set :@columns, columns
|
2033
|
+
new_fam
|
2034
|
+
end
|
2035
|
+
|
2036
|
+
##
|
2037
|
+
# @private
|
2038
|
+
def freeze
|
2039
|
+
@columns.map(&:freeze!)
|
2040
|
+
@columns.freeze!
|
2041
|
+
super
|
2042
|
+
end
|
2043
|
+
|
2044
|
+
protected
|
2045
|
+
|
2046
|
+
def frozen_check!
|
2047
|
+
return unless frozen?
|
2048
|
+
raise ArgumentError, "Cannot modify external data source when frozen"
|
2049
|
+
end
|
2050
|
+
end
|
2051
|
+
|
2052
|
+
##
|
2053
|
+
# # BigtableSource::Column
|
2054
|
+
#
|
2055
|
+
# A Bigtable column to expose in the table schema along with its
|
2056
|
+
# types.
|
2057
|
+
#
|
2058
|
+
# @example
|
2059
|
+
# require "google/cloud/bigquery"
|
2060
|
+
#
|
2061
|
+
# bigquery = Google::Cloud::Bigquery.new
|
2062
|
+
#
|
2063
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
2064
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
2065
|
+
# bt.rowkey_as_string = true
|
2066
|
+
# bt.add_family "user" do |u|
|
2067
|
+
# u.add_string "name"
|
2068
|
+
# u.add_string "email"
|
2069
|
+
# u.add_integer "age"
|
2070
|
+
# u.add_boolean "active"
|
2071
|
+
# end
|
2072
|
+
# end
|
2073
|
+
#
|
2074
|
+
# data = bigquery.query "SELECT * FROM my_ext_table",
|
2075
|
+
# external: { my_ext_table: bigtable_table }
|
2076
|
+
#
|
2077
|
+
# # Iterate over the first page of results
|
2078
|
+
# data.each do |row|
|
2079
|
+
# puts row[:name]
|
2080
|
+
# end
|
2081
|
+
# # Retrieve the next page of results
|
2082
|
+
# data = data.next if data.next?
|
2083
|
+
#
|
2084
|
+
class Column
|
2085
|
+
##
|
2086
|
+
# @private Create an empty BigtableSource::Column object.
|
2087
|
+
def initialize
|
2088
|
+
@gapi = Google::Apis::BigqueryV2::BigtableColumn.new
|
2089
|
+
end
|
2090
|
+
|
2091
|
+
##
|
2092
|
+
# Qualifier of the column. Columns in the parent column family that
|
2093
|
+
# has this exact qualifier are exposed as `.` field. If the
|
2094
|
+
# qualifier is valid UTF-8 string, it will be represented as a UTF-8
|
2095
|
+
# string. Otherwise, it will represented as a ASCII-8BIT string.
|
2096
|
+
#
|
2097
|
+
# If the qualifier is not a valid BigQuery field identifier (does
|
2098
|
+
# not match `[a-zA-Z][a-zA-Z0-9_]*`) a valid identifier must be
|
2099
|
+
# provided as `field_name`.
|
2100
|
+
#
|
2101
|
+
# @return [String]
|
2102
|
+
#
|
2103
|
+
# @example
|
2104
|
+
# require "google/cloud/bigquery"
|
2105
|
+
#
|
2106
|
+
# bigquery = Google::Cloud::Bigquery.new
|
2107
|
+
#
|
2108
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
2109
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
2110
|
+
# bt.add_family "user" do |u|
|
2111
|
+
# u.add_string "name" do |col|
|
2112
|
+
# col.qualifier # "user"
|
2113
|
+
# col.qualifier = "User"
|
2114
|
+
# col.qualifier # "User"
|
2115
|
+
# end
|
2116
|
+
# end
|
2117
|
+
# end
|
2118
|
+
#
|
2119
|
+
def qualifier
|
2120
|
+
@gapi.qualifier_string || Base64.strict_decode64(@gapi.qualifier_encoded.to_s)
|
2121
|
+
end
|
2122
|
+
|
2123
|
+
##
|
2124
|
+
# Set the qualifier of the column. Columns in the parent column
|
2125
|
+
# family that has this exact qualifier are exposed as `.` field.
|
2126
|
+
# Values that are valid UTF-8 strings will be treated as such. All
|
2127
|
+
# other values will be treated as `BINARY`.
|
2128
|
+
#
|
2129
|
+
# @param [String] new_qualifier New qualifier value
|
2130
|
+
#
|
2131
|
+
# @example
|
2132
|
+
# require "google/cloud/bigquery"
|
2133
|
+
#
|
2134
|
+
# bigquery = Google::Cloud::Bigquery.new
|
2135
|
+
#
|
2136
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
2137
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
2138
|
+
# bt.add_family "user" do |u|
|
2139
|
+
# u.add_string "name" do |col|
|
2140
|
+
# col.qualifier # "user"
|
2141
|
+
# col.qualifier = "User"
|
2142
|
+
# col.qualifier # "User"
|
2143
|
+
# end
|
2144
|
+
# end
|
2145
|
+
# end
|
2146
|
+
#
|
2147
|
+
def qualifier= new_qualifier
|
2148
|
+
frozen_check!
|
2149
|
+
raise ArgumentError if new_qualifier.nil?
|
2150
|
+
|
2151
|
+
utf8_qualifier = new_qualifier.encode Encoding::UTF_8
|
2152
|
+
if utf8_qualifier.valid_encoding?
|
2153
|
+
@gapi.qualifier_string = utf8_qualifier
|
2154
|
+
if @gapi.instance_variables.include? :@qualifier_encoded
|
2155
|
+
@gapi.remove_instance_variable :@qualifier_encoded
|
2156
|
+
end
|
2157
|
+
else
|
2158
|
+
@gapi.qualifier_encoded = Base64.strict_encode64 new_qualifier
|
2159
|
+
if @gapi.instance_variables.include? :@qualifier_string
|
2160
|
+
@gapi.remove_instance_variable :@qualifier_string
|
2161
|
+
end
|
2162
|
+
end
|
2163
|
+
rescue EncodingError
|
2164
|
+
@gapi.qualifier_encoded = Base64.strict_encode64 new_qualifier
|
2165
|
+
@gapi.remove_instance_variable :@qualifier_string if @gapi.instance_variables.include? :@qualifier_string
|
2166
|
+
end
|
2167
|
+
|
2168
|
+
##
|
2169
|
+
# The encoding of the values when the type is not `STRING`.
|
2170
|
+
#
|
2171
|
+
# @return [String]
|
2172
|
+
#
|
2173
|
+
# @example
|
2174
|
+
# require "google/cloud/bigquery"
|
2175
|
+
#
|
2176
|
+
# bigquery = Google::Cloud::Bigquery.new
|
2177
|
+
#
|
2178
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
2179
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
2180
|
+
# bt.add_family "user" do |u|
|
2181
|
+
# u.add_bytes "name" do |col|
|
2182
|
+
# col.encoding = "TEXT"
|
2183
|
+
# col.encoding # "TEXT"
|
2184
|
+
# end
|
2185
|
+
# end
|
2186
|
+
# end
|
2187
|
+
#
|
2188
|
+
def encoding
|
2189
|
+
@gapi.encoding
|
2190
|
+
end
|
2191
|
+
|
2192
|
+
##
|
2193
|
+
# Set the encoding of the values when the type is not `STRING`.
|
2194
|
+
# Acceptable encoding values are:
|
2195
|
+
#
|
2196
|
+
# * `TEXT` - indicates values are alphanumeric text strings.
|
2197
|
+
# * `BINARY` - indicates values are encoded using HBase
|
2198
|
+
# `Bytes.toBytes` family of functions. This can be overridden on a
|
2199
|
+
# column.
|
2200
|
+
#
|
2201
|
+
# @param [String] new_encoding New encoding value
|
2202
|
+
#
|
2203
|
+
# @example
|
2204
|
+
# require "google/cloud/bigquery"
|
2205
|
+
#
|
2206
|
+
# bigquery = Google::Cloud::Bigquery.new
|
2207
|
+
#
|
2208
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
2209
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
2210
|
+
# bt.add_family "user" do |u|
|
2211
|
+
# u.add_bytes "name" do |col|
|
2212
|
+
# col.encoding = "TEXT"
|
2213
|
+
# col.encoding # "TEXT"
|
2214
|
+
# end
|
2215
|
+
# end
|
2216
|
+
# end
|
2217
|
+
#
|
2218
|
+
def encoding= new_encoding
|
2219
|
+
frozen_check!
|
2220
|
+
@gapi.encoding = new_encoding
|
2221
|
+
end
|
2222
|
+
|
2223
|
+
##
|
2224
|
+
# If the qualifier is not a valid BigQuery field identifier (does
|
2225
|
+
# not match `[a-zA-Z][a-zA-Z0-9_]*`) a valid identifier must be
|
2226
|
+
# provided as the column field name and is used as field name in
|
2227
|
+
# queries.
|
2228
|
+
#
|
2229
|
+
# @return [String]
|
2230
|
+
#
|
2231
|
+
# @example
|
2232
|
+
# require "google/cloud/bigquery"
|
2233
|
+
#
|
2234
|
+
# bigquery = Google::Cloud::Bigquery.new
|
2235
|
+
#
|
2236
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
2237
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
2238
|
+
# bt.add_family "user" do |u|
|
2239
|
+
# u.add_string "001_name", as: "user" do |col|
|
2240
|
+
# col.field_name # "user"
|
2241
|
+
# col.field_name = "User"
|
2242
|
+
# col.field_name # "User"
|
2243
|
+
# end
|
2244
|
+
# end
|
2245
|
+
# end
|
2246
|
+
#
|
2247
|
+
def field_name
|
2248
|
+
@gapi.field_name
|
2249
|
+
end
|
2250
|
+
|
2251
|
+
##
|
2252
|
+
# Sets the identifier to be used as the column field name in queries
|
2253
|
+
# when the qualifier is not a valid BigQuery field identifier (does
|
2254
|
+
# not match `[a-zA-Z][a-zA-Z0-9_]*`).
|
2255
|
+
#
|
2256
|
+
# @param [String] new_field_name New field_name value
|
2257
|
+
#
|
2258
|
+
# @example
|
2259
|
+
# require "google/cloud/bigquery"
|
2260
|
+
#
|
2261
|
+
# bigquery = Google::Cloud::Bigquery.new
|
2262
|
+
#
|
2263
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
2264
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
2265
|
+
# bt.add_family "user" do |u|
|
2266
|
+
# u.add_string "001_name", as: "user" do |col|
|
2267
|
+
# col.field_name # "user"
|
2268
|
+
# col.field_name = "User"
|
2269
|
+
# col.field_name # "User"
|
2270
|
+
# end
|
2271
|
+
# end
|
2272
|
+
# end
|
2273
|
+
#
|
2274
|
+
def field_name= new_field_name
|
2275
|
+
frozen_check!
|
2276
|
+
@gapi.field_name = new_field_name
|
2277
|
+
end
|
2278
|
+
|
2279
|
+
##
|
2280
|
+
# Whether only the latest version of value in this column are
|
2281
|
+
# exposed. Can also be set at the column family level. However, this
|
2282
|
+
# value takes precedence when set at both levels.
|
2283
|
+
#
|
2284
|
+
# @return [Boolean]
|
2285
|
+
#
|
2286
|
+
# @example
|
2287
|
+
# require "google/cloud/bigquery"
|
2288
|
+
#
|
2289
|
+
# bigquery = Google::Cloud::Bigquery.new
|
2290
|
+
#
|
2291
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
2292
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
2293
|
+
# bt.add_family "user" do |u|
|
2294
|
+
# u.add_string "name" do |col|
|
2295
|
+
# col.latest = true
|
2296
|
+
# col.latest # true
|
2297
|
+
# end
|
2298
|
+
# end
|
2299
|
+
# end
|
2300
|
+
#
|
2301
|
+
def latest
|
2302
|
+
@gapi.only_read_latest
|
2303
|
+
end
|
2304
|
+
|
2305
|
+
##
|
2306
|
+
# Set whether only the latest version of value in this column are
|
2307
|
+
# exposed. Can also be set at the column family level. However, this
|
2308
|
+
# value takes precedence when set at both levels.
|
2309
|
+
#
|
2310
|
+
# @param [Boolean] new_latest New latest value
|
2311
|
+
#
|
2312
|
+
# @example
|
2313
|
+
# require "google/cloud/bigquery"
|
2314
|
+
#
|
2315
|
+
# bigquery = Google::Cloud::Bigquery.new
|
2316
|
+
#
|
2317
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
2318
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
2319
|
+
# bt.add_family "user" do |u|
|
2320
|
+
# u.add_string "name" do |col|
|
2321
|
+
# col.latest = true
|
2322
|
+
# col.latest # true
|
2323
|
+
# end
|
2324
|
+
# end
|
2325
|
+
# end
|
2326
|
+
#
|
2327
|
+
def latest= new_latest
|
2328
|
+
frozen_check!
|
2329
|
+
@gapi.only_read_latest = new_latest
|
2330
|
+
end
|
2331
|
+
|
2332
|
+
##
|
2333
|
+
# The type to convert the value in cells of this column. The values
|
2334
|
+
# are expected to be encoded using HBase `Bytes.toBytes` function
|
2335
|
+
# when using the `BINARY` encoding value. The following BigQuery
|
2336
|
+
# types are allowed:
|
2337
|
+
#
|
2338
|
+
# * `BYTES`
|
2339
|
+
# * `STRING`
|
2340
|
+
# * `INTEGER`
|
2341
|
+
# * `FLOAT`
|
2342
|
+
# * `BOOLEAN`
|
2343
|
+
#
|
2344
|
+
# Default type is `BYTES`. Can also be set at the column family
|
2345
|
+
# level. However, this value takes precedence when set at both
|
2346
|
+
# levels.
|
2347
|
+
#
|
2348
|
+
# @return [String]
|
2349
|
+
#
|
2350
|
+
# @example
|
2351
|
+
# require "google/cloud/bigquery"
|
2352
|
+
#
|
2353
|
+
# bigquery = Google::Cloud::Bigquery.new
|
2354
|
+
#
|
2355
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
2356
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
2357
|
+
# bt.add_family "user" do |u|
|
2358
|
+
# u.add_string "name" do |col|
|
2359
|
+
# col.type # "STRING"
|
2360
|
+
# end
|
2361
|
+
# end
|
2362
|
+
# end
|
2363
|
+
#
|
2364
|
+
def type
|
2365
|
+
@gapi.type
|
2366
|
+
end
|
2367
|
+
|
2368
|
+
##
|
2369
|
+
# Set the type to convert the value in cells of this column. The
|
2370
|
+
# values are expected to be encoded using HBase `Bytes.toBytes`
|
2371
|
+
# function when using the `BINARY` encoding value. The following
|
2372
|
+
# BigQuery types are allowed:
|
2373
|
+
#
|
2374
|
+
# * `BYTES`
|
2375
|
+
# * `STRING`
|
2376
|
+
# * `INTEGER`
|
2377
|
+
# * `FLOAT`
|
2378
|
+
# * `BOOLEAN`
|
2379
|
+
#
|
2380
|
+
# Default type is `BYTES`. Can also be set at the column family
|
2381
|
+
# level. However, this value takes precedence when set at both
|
2382
|
+
# levels.
|
2383
|
+
#
|
2384
|
+
# @param [String] new_type New type value
|
2385
|
+
#
|
2386
|
+
# @example
|
2387
|
+
# require "google/cloud/bigquery"
|
2388
|
+
#
|
2389
|
+
# bigquery = Google::Cloud::Bigquery.new
|
2390
|
+
#
|
2391
|
+
# bigtable_url = "https://googleapis.com/bigtable/projects/..."
|
2392
|
+
# bigtable_table = bigquery.external bigtable_url do |bt|
|
2393
|
+
# bt.add_family "user" do |u|
|
2394
|
+
# u.add_string "name" do |col|
|
2395
|
+
# col.type # "STRING"
|
2396
|
+
# col.type = "BYTES"
|
2397
|
+
# col.type # "BYTES"
|
2398
|
+
# end
|
2399
|
+
# end
|
2400
|
+
# end
|
2401
|
+
#
|
2402
|
+
def type= new_type
|
2403
|
+
frozen_check!
|
2404
|
+
@gapi.type = new_type
|
2405
|
+
end
|
2406
|
+
|
2407
|
+
##
|
2408
|
+
# @private Google API Client object.
|
2409
|
+
def to_gapi
|
2410
|
+
@gapi
|
2411
|
+
end
|
2412
|
+
|
2413
|
+
##
|
2414
|
+
# @private Google API Client object.
|
2415
|
+
def self.from_gapi gapi
|
2416
|
+
new_col = new
|
2417
|
+
new_col.instance_variable_set :@gapi, gapi
|
2418
|
+
new_col
|
2419
|
+
end
|
2420
|
+
|
2421
|
+
protected
|
2422
|
+
|
2423
|
+
def frozen_check!
|
2424
|
+
return unless frozen?
|
2425
|
+
raise ArgumentError, "Cannot modify external data source when frozen"
|
2426
|
+
end
|
2427
|
+
end
|
2428
|
+
end
|
2429
|
+
end
|
2430
|
+
end
|
2431
|
+
end
|
2432
|
+
end
|