google-cloud-bigquery 1.36.0 → 1.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d23511050f9602220787610566c2f1f62ed37ad3c014e9708e16a05bf4a079a1
|
|
4
|
+
data.tar.gz: 5a61cffca84ffd8a1561f28d6d2d545d7451389f4a363279bea40f907c1ec568
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4bbdca09750d11aabed55d8868b9b70501def5a8a12b35f4e0b92744c7d2a53e9ba503384771ccad85eb32ddc133b840d30a8434efc82260f4ff2fab2fbec6ac
|
|
7
|
+
data.tar.gz: 8990512c43041ad7b3b2e4d87c64d22b080a6a0bbc8d648fe09942504b2bc607a910809688b4f13410fb5c8fe2222f4b12ea8a613a9ac52864835319a1b56c66
|
data/CHANGELOG.md
CHANGED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# Copyright 2021 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
require "google/apis/bigquery_v2"
|
|
17
|
+
|
|
18
|
+
module Google
|
|
19
|
+
module Cloud
|
|
20
|
+
module Bigquery
|
|
21
|
+
module External
|
|
22
|
+
##
|
|
23
|
+
# # AvroSource
|
|
24
|
+
#
|
|
25
|
+
# {External::AvroSource} is a subclass of {External::DataSource} and
|
|
26
|
+
# represents a Avro external data source that can be queried
|
|
27
|
+
# from directly, even though the data is not stored in BigQuery. Instead
|
|
28
|
+
# of loading or streaming the data, this object references the external
|
|
29
|
+
# data source.
|
|
30
|
+
#
|
|
31
|
+
# @example
|
|
32
|
+
# require "google/cloud/bigquery"
|
|
33
|
+
#
|
|
34
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
35
|
+
#
|
|
36
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
|
37
|
+
# avro_table = bigquery.external avro_url do |avro|
|
|
38
|
+
# avro.use_avro_logical_types = 1
|
|
39
|
+
# end
|
|
40
|
+
#
|
|
41
|
+
# data = bigquery.query "SELECT * FROM my_ext_table",
|
|
42
|
+
# external: { my_ext_table: avro_table }
|
|
43
|
+
#
|
|
44
|
+
# # Iterate over the first page of results
|
|
45
|
+
# data.each do |row|
|
|
46
|
+
# puts row[:name]
|
|
47
|
+
# end
|
|
48
|
+
# # Retrieve the next page of results
|
|
49
|
+
# data = data.next if data.next?
|
|
50
|
+
#
|
|
51
|
+
class AvroSource < External::DataSource
|
|
52
|
+
##
|
|
53
|
+
# @private Create an empty AvroSource object.
|
|
54
|
+
def initialize
|
|
55
|
+
super
|
|
56
|
+
@gapi.avro_options = Google::Apis::BigqueryV2::AvroOptions.new
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
##
|
|
60
|
+
# Indicates whether to interpret logical types as the corresponding BigQuery data type (for example,
|
|
61
|
+
# `TIMESTAMP`), instead of using the raw type (for example, `INTEGER`).
|
|
62
|
+
#
|
|
63
|
+
# @return [Boolean]
|
|
64
|
+
#
|
|
65
|
+
# @example
|
|
66
|
+
# require "google/cloud/bigquery"
|
|
67
|
+
#
|
|
68
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
69
|
+
#
|
|
70
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
|
71
|
+
# avro_table = bigquery.external avro_url do |avro|
|
|
72
|
+
# avro.use_avro_logical_types = true
|
|
73
|
+
# end
|
|
74
|
+
#
|
|
75
|
+
# avro_table.use_avro_logical_types #=> true
|
|
76
|
+
#
|
|
77
|
+
def use_avro_logical_types
|
|
78
|
+
@gapi.avro_options.use_avro_logical_types
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
##
|
|
82
|
+
# Sets whether to interpret logical types as the corresponding BigQuery data type (for example, `TIMESTAMP`),
|
|
83
|
+
# instead of using the raw type (for example, `INTEGER`).
|
|
84
|
+
#
|
|
85
|
+
# @param [Boolean] new_use_avro_logical_types The new `use_avro_logical_types` value.
|
|
86
|
+
#
|
|
87
|
+
# @example
|
|
88
|
+
# require "google/cloud/bigquery"
|
|
89
|
+
#
|
|
90
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
91
|
+
#
|
|
92
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
|
93
|
+
# avro_table = bigquery.external avro_url do |avro|
|
|
94
|
+
# avro.use_avro_logical_types = true
|
|
95
|
+
# end
|
|
96
|
+
#
|
|
97
|
+
# avro_table.use_avro_logical_types #=> true
|
|
98
|
+
#
|
|
99
|
+
def use_avro_logical_types= new_use_avro_logical_types
|
|
100
|
+
frozen_check!
|
|
101
|
+
@gapi.avro_options.use_avro_logical_types = new_use_avro_logical_types
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
@@ -37,7 +37,7 @@ module Google
|
|
|
37
37
|
#
|
|
38
38
|
# bigquery = Google::Cloud::Bigquery.new
|
|
39
39
|
#
|
|
40
|
-
# avro_url = "gs://bucket/path/to
|
|
40
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
|
41
41
|
# avro_table = bigquery.external avro_url do |avro|
|
|
42
42
|
# avro.autodetect = true
|
|
43
43
|
# end
|
|
@@ -174,7 +174,7 @@ module Google
|
|
|
174
174
|
#
|
|
175
175
|
# bigquery = Google::Cloud::Bigquery.new
|
|
176
176
|
#
|
|
177
|
-
# avro_url = "gs://bucket/path/to
|
|
177
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
|
178
178
|
# avro_table = bigquery.external avro_url
|
|
179
179
|
#
|
|
180
180
|
# avro_table.format #=> "AVRO"
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
require "google/cloud/bigquery/external/data_source"
|
|
17
|
+
require "google/cloud/bigquery/external/avro_source"
|
|
17
18
|
require "google/cloud/bigquery/external/bigtable_source"
|
|
18
19
|
require "google/cloud/bigquery/external/csv_source"
|
|
19
20
|
require "google/cloud/bigquery/external/json_source"
|
|
@@ -101,28 +102,28 @@ module Google
|
|
|
101
102
|
# @private Determine source_format from inputs
|
|
102
103
|
def self.source_format_for urls, format
|
|
103
104
|
val = {
|
|
104
|
-
"csv" => "CSV",
|
|
105
105
|
"avro" => "AVRO",
|
|
106
|
-
"
|
|
107
|
-
"
|
|
108
|
-
"sheets" => "GOOGLE_SHEETS",
|
|
109
|
-
"google_sheets" => "GOOGLE_SHEETS",
|
|
110
|
-
"datastore" => "DATASTORE_BACKUP",
|
|
106
|
+
"bigtable" => "BIGTABLE",
|
|
107
|
+
"csv" => "CSV",
|
|
111
108
|
"backup" => "DATASTORE_BACKUP",
|
|
109
|
+
"datastore" => "DATASTORE_BACKUP",
|
|
112
110
|
"datastore_backup" => "DATASTORE_BACKUP",
|
|
113
|
-
"
|
|
111
|
+
"sheets" => "GOOGLE_SHEETS",
|
|
112
|
+
"google_sheets" => "GOOGLE_SHEETS",
|
|
113
|
+
"json" => "NEWLINE_DELIMITED_JSON",
|
|
114
|
+
"newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
|
|
114
115
|
"orc" => "ORC",
|
|
115
116
|
"parquet" => "PARQUET"
|
|
116
117
|
}[format.to_s.downcase]
|
|
117
118
|
return val unless val.nil?
|
|
118
119
|
Array(urls).each do |url|
|
|
119
|
-
return "CSV" if url.end_with? ".csv"
|
|
120
|
-
return "NEWLINE_DELIMITED_JSON" if url.end_with? ".json"
|
|
121
|
-
return "PARQUET" if url.end_with? ".parquet"
|
|
122
120
|
return "AVRO" if url.end_with? ".avro"
|
|
121
|
+
return "BIGTABLE" if url.start_with? "https://googleapis.com/bigtable/projects/"
|
|
122
|
+
return "CSV" if url.end_with? ".csv"
|
|
123
123
|
return "DATASTORE_BACKUP" if url.end_with? ".backup_info"
|
|
124
124
|
return "GOOGLE_SHEETS" if url.start_with? "https://docs.google.com/spreadsheets/"
|
|
125
|
-
return "
|
|
125
|
+
return "NEWLINE_DELIMITED_JSON" if url.end_with? ".json"
|
|
126
|
+
return "PARQUET" if url.end_with? ".parquet"
|
|
126
127
|
end
|
|
127
128
|
nil
|
|
128
129
|
end
|
|
@@ -131,13 +132,14 @@ module Google
|
|
|
131
132
|
# @private Determine table class from source_format
|
|
132
133
|
def self.table_class_for format
|
|
133
134
|
case format
|
|
135
|
+
when "AVRO" then External::AvroSource
|
|
136
|
+
when "BIGTABLE" then External::BigtableSource
|
|
134
137
|
when "CSV" then External::CsvSource
|
|
138
|
+
when "GOOGLE_SHEETS" then External::SheetsSource
|
|
135
139
|
when "NEWLINE_DELIMITED_JSON" then External::JsonSource
|
|
136
140
|
when "PARQUET" then External::ParquetSource
|
|
137
|
-
when "GOOGLE_SHEETS" then External::SheetsSource
|
|
138
|
-
when "BIGTABLE" then External::BigtableSource
|
|
139
141
|
else
|
|
140
|
-
#
|
|
142
|
+
# DATASTORE_BACKUP, ORC
|
|
141
143
|
External::DataSource
|
|
142
144
|
end
|
|
143
145
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: google-cloud-bigquery
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.37.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Mike Moore
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2021-
|
|
12
|
+
date: 2021-10-21 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: concurrent-ruby
|
|
@@ -270,6 +270,7 @@ files:
|
|
|
270
270
|
- lib/google/cloud/bigquery/dataset/list.rb
|
|
271
271
|
- lib/google/cloud/bigquery/encryption_configuration.rb
|
|
272
272
|
- lib/google/cloud/bigquery/external.rb
|
|
273
|
+
- lib/google/cloud/bigquery/external/avro_source.rb
|
|
273
274
|
- lib/google/cloud/bigquery/external/bigtable_source.rb
|
|
274
275
|
- lib/google/cloud/bigquery/external/bigtable_source/column.rb
|
|
275
276
|
- lib/google/cloud/bigquery/external/bigtable_source/column_family.rb
|