google-cloud-bigquery 1.34.0 → 1.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +36 -0
- data/lib/google/cloud/bigquery/convert.rb +3 -0
- data/lib/google/cloud/bigquery/dataset.rb +28 -21
- data/lib/google/cloud/bigquery/external/avro_source.rb +107 -0
- data/lib/google/cloud/bigquery/external/data_source.rb +2 -2
- data/lib/google/cloud/bigquery/external.rb +16 -14
- data/lib/google/cloud/bigquery/job.rb +33 -2
- data/lib/google/cloud/bigquery/load_job.rb +48 -12
- data/lib/google/cloud/bigquery/model.rb +8 -8
- data/lib/google/cloud/bigquery/project.rb +26 -20
- data/lib/google/cloud/bigquery/query_job.rb +11 -6
- data/lib/google/cloud/bigquery/routine.rb +2 -2
- data/lib/google/cloud/bigquery/schema/field.rb +114 -29
- data/lib/google/cloud/bigquery/schema.rb +34 -12
- data/lib/google/cloud/bigquery/service.rb +9 -1
- data/lib/google/cloud/bigquery/table/async_inserter.rb +1 -0
- data/lib/google/cloud/bigquery/table.rb +62 -28
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d23511050f9602220787610566c2f1f62ed37ad3c014e9708e16a05bf4a079a1
|
|
4
|
+
data.tar.gz: 5a61cffca84ffd8a1561f28d6d2d545d7451389f4a363279bea40f907c1ec568
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4bbdca09750d11aabed55d8868b9b70501def5a8a12b35f4e0b92744c7d2a53e9ba503384771ccad85eb32ddc133b840d30a8434efc82260f4ff2fab2fbec6ac
|
|
7
|
+
data.tar.gz: 8990512c43041ad7b3b2e4d87c64d22b080a6a0bbc8d648fe09942504b2bc607a910809688b4f13410fb5c8fe2222f4b12ea8a613a9ac52864835319a1b56c66
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,41 @@
|
|
|
1
1
|
# Release History
|
|
2
2
|
|
|
3
|
+
### 1.37.0 / 2021-10-21
|
|
4
|
+
|
|
5
|
+
#### Features
|
|
6
|
+
|
|
7
|
+
* Add support for Avro options to external data sources
|
|
8
|
+
* Add External::AvroSource
|
|
9
|
+
|
|
10
|
+
### 1.36.0 / 2021-09-22
|
|
11
|
+
|
|
12
|
+
#### Features
|
|
13
|
+
|
|
14
|
+
* Add Job#delete
|
|
15
|
+
|
|
16
|
+
#### Bug Fixes
|
|
17
|
+
|
|
18
|
+
* Add precision and scale to Field#add_field
|
|
19
|
+
|
|
20
|
+
### 1.35.1 / 2021-09-14
|
|
21
|
+
|
|
22
|
+
#### Documentation
|
|
23
|
+
|
|
24
|
+
* Wrap character class regex in backticks
|
|
25
|
+
|
|
26
|
+
### 1.35.0 / 2021-08-12
|
|
27
|
+
|
|
28
|
+
#### Features
|
|
29
|
+
|
|
30
|
+
* Add GEOGRAPHY schema helpers
|
|
31
|
+
* Add LoadJob#geography
|
|
32
|
+
* Add Schema::Field#geography
|
|
33
|
+
* Add Table::Updater#geography
|
|
34
|
+
* Add support for GEOGRAPHY type
|
|
35
|
+
* Add Schema#geography
|
|
36
|
+
* Add support for multistatement transaction statistics in jobs
|
|
37
|
+
* Add Job#transaction_id
|
|
38
|
+
|
|
3
39
|
### 1.34.0 / 2021-07-20
|
|
4
40
|
|
|
5
41
|
#### Features
|
|
@@ -38,6 +38,7 @@ module Google
|
|
|
38
38
|
# | `STRING` | `String` | |
|
|
39
39
|
# | `DATETIME` | `DateTime` | `DATETIME` does not support time zone. |
|
|
40
40
|
# | `DATE` | `Date` | |
|
|
41
|
+
# | `GEOGRAPHY` | `String` | |
|
|
41
42
|
# | `TIMESTAMP` | `Time` | |
|
|
42
43
|
# | `TIME` | `Google::Cloud::BigQuery::Time` | |
|
|
43
44
|
# | `BYTES` | `File`, `IO`, `StringIO`, or similar | |
|
|
@@ -102,6 +103,8 @@ module Google
|
|
|
102
103
|
::Time.parse("#{value[:v]} UTC").to_datetime
|
|
103
104
|
elsif field.type == "DATE"
|
|
104
105
|
Date.parse value[:v]
|
|
106
|
+
elsif field.type == "GEOGRAPHY"
|
|
107
|
+
String value[:v]
|
|
105
108
|
else
|
|
106
109
|
value[:v]
|
|
107
110
|
end
|
|
@@ -69,8 +69,8 @@ module Google
|
|
|
69
69
|
##
|
|
70
70
|
# A unique ID for this dataset, without the project name.
|
|
71
71
|
#
|
|
72
|
-
# @return [String] The ID must contain only letters (
|
|
73
|
-
# (0-9), or underscores (_). The maximum length is 1,024 characters.
|
|
72
|
+
# @return [String] The ID must contain only letters (`[A-Za-z]`), numbers
|
|
73
|
+
# (`[0-9]`), or underscores (`_`). The maximum length is 1,024 characters.
|
|
74
74
|
#
|
|
75
75
|
# @!group Attributes
|
|
76
76
|
#
|
|
@@ -501,7 +501,7 @@ module Google
|
|
|
501
501
|
# you can pass the table's schema as a hash (see example.)
|
|
502
502
|
#
|
|
503
503
|
# @param [String] table_id The ID of the table. The ID must contain only
|
|
504
|
-
# letters (
|
|
504
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`). The maximum
|
|
505
505
|
# length is 1,024 characters.
|
|
506
506
|
# @param [String] name A descriptive name for the table.
|
|
507
507
|
# @param [String] description A user-friendly description of the table.
|
|
@@ -630,7 +630,7 @@ module Google
|
|
|
630
630
|
# @see https://cloud.google.com/bigquery/docs/views Creating views
|
|
631
631
|
#
|
|
632
632
|
# @param [String] table_id The ID of the view table. The ID must contain
|
|
633
|
-
# only letters (
|
|
633
|
+
# only letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`). The
|
|
634
634
|
# maximum length is 1,024 characters.
|
|
635
635
|
# @param [String] query The query that BigQuery executes when the view
|
|
636
636
|
# is referenced.
|
|
@@ -726,8 +726,8 @@ module Google
|
|
|
726
726
|
#
|
|
727
727
|
# @see https://cloud.google.com/bigquery/docs/materialized-views-intro Introduction to materialized views
|
|
728
728
|
#
|
|
729
|
-
# @param [String] table_id The ID of the materialized view table. The ID must contain only letters (
|
|
730
|
-
# numbers (0-9), or underscores (_). The maximum length is 1,024 characters.
|
|
729
|
+
# @param [String] table_id The ID of the materialized view table. The ID must contain only letters (`[A-Za-z]`),
|
|
730
|
+
# numbers (`[0-9]`), or underscores (`_`). The maximum length is 1,024 characters.
|
|
731
731
|
# @param [String] query The query that BigQuery executes when the materialized view is referenced.
|
|
732
732
|
# @param [String] name A descriptive name for the table.
|
|
733
733
|
# @param [String] description A user-friendly description of the table.
|
|
@@ -954,7 +954,7 @@ module Google
|
|
|
954
954
|
# {Routine::Updater#description=}.
|
|
955
955
|
#
|
|
956
956
|
# @param [String] routine_id The ID of the routine. The ID must contain only
|
|
957
|
-
# letters (
|
|
957
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`). The maximum length
|
|
958
958
|
# is 256 characters.
|
|
959
959
|
# @yield [routine] A block for setting properties on the routine.
|
|
960
960
|
# @yieldparam [Google::Cloud::Bigquery::Routine::Updater] routine An updater to set additional properties on the
|
|
@@ -1138,7 +1138,7 @@ module Google
|
|
|
1138
1138
|
# use named query parameters. When set, `legacy_sql` will automatically be set to false and `standard_sql` to
|
|
1139
1139
|
# true.
|
|
1140
1140
|
#
|
|
1141
|
-
#
|
|
1141
|
+
# BigQuery types are converted from Ruby types as follows:
|
|
1142
1142
|
#
|
|
1143
1143
|
# | BigQuery | Ruby | Notes |
|
|
1144
1144
|
# |--------------|--------------------------------------|----------------------------------------------------|
|
|
@@ -1146,10 +1146,11 @@ module Google
|
|
|
1146
1146
|
# | `INT64` | `Integer` | |
|
|
1147
1147
|
# | `FLOAT64` | `Float` | |
|
|
1148
1148
|
# | `NUMERIC` | `BigDecimal` | `BigDecimal` values will be rounded to scale 9. |
|
|
1149
|
-
# | `BIGNUMERIC` |
|
|
1149
|
+
# | `BIGNUMERIC` | `BigDecimal` | NOT AUTOMATIC: Must be mapped using `types`, below.|
|
|
1150
1150
|
# | `STRING` | `String` | |
|
|
1151
1151
|
# | `DATETIME` | `DateTime` | `DATETIME` does not support time zone. |
|
|
1152
1152
|
# | `DATE` | `Date` | |
|
|
1153
|
+
# | `GEOGRAPHY` | `String` (WKT or GeoJSON) | NOT AUTOMATIC: Must be mapped using `types`, below.|
|
|
1153
1154
|
# | `TIMESTAMP` | `Time` | |
|
|
1154
1155
|
# | `TIME` | `Google::Cloud::BigQuery::Time` | |
|
|
1155
1156
|
# | `BYTES` | `File`, `IO`, `StringIO`, or similar | |
|
|
@@ -1157,7 +1158,8 @@ module Google
|
|
|
1157
1158
|
# | `STRUCT` | `Hash` | Hash keys may be strings or symbols. |
|
|
1158
1159
|
#
|
|
1159
1160
|
# See [Data Types](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types) for an overview
|
|
1160
|
-
# of each BigQuery data type, including allowed values.
|
|
1161
|
+
# of each BigQuery data type, including allowed values. For the `GEOGRAPHY` type, see [Working with BigQuery
|
|
1162
|
+
# GIS data](https://cloud.google.com/bigquery/docs/gis-data).
|
|
1161
1163
|
# @param [Array, Hash] types Standard SQL only. Types of the SQL parameters in `params`. It is not always
|
|
1162
1164
|
# possible to infer the right SQL type from a value in `params`. In these cases, `types` must be used to
|
|
1163
1165
|
# specify the SQL type for these values.
|
|
@@ -1174,6 +1176,7 @@ module Google
|
|
|
1174
1176
|
# * `:STRING`
|
|
1175
1177
|
# * `:DATETIME`
|
|
1176
1178
|
# * `:DATE`
|
|
1179
|
+
# * `:GEOGRAPHY`
|
|
1177
1180
|
# * `:TIMESTAMP`
|
|
1178
1181
|
# * `:TIME`
|
|
1179
1182
|
# * `:BYTES`
|
|
@@ -1246,8 +1249,8 @@ module Google
|
|
|
1246
1249
|
# (without incurring a charge). Optional. If unspecified, this will be
|
|
1247
1250
|
# set to your project default.
|
|
1248
1251
|
# @param [String] job_id A user-defined ID for the query job. The ID
|
|
1249
|
-
# must contain only letters (
|
|
1250
|
-
# (_), or dashes (
|
|
1252
|
+
# must contain only letters (`[A-Za-z]`), numbers (`[0-9]`), underscores
|
|
1253
|
+
# (`_`), or dashes (`-`). The maximum length is 1,024 characters. If
|
|
1251
1254
|
# `job_id` is provided, then `prefix` will not be used.
|
|
1252
1255
|
#
|
|
1253
1256
|
# See [Generating a job
|
|
@@ -1256,8 +1259,8 @@ module Google
|
|
|
1256
1259
|
# prepended to a generated value to produce a unique job ID. For
|
|
1257
1260
|
# example, the prefix `daily_import_job_` can be given to generate a
|
|
1258
1261
|
# job ID such as `daily_import_job_12vEDtMQ0mbp1Mo5Z7mzAFQJZazh`. The
|
|
1259
|
-
# prefix must contain only letters (
|
|
1260
|
-
# underscores (_), or dashes (
|
|
1262
|
+
# prefix must contain only letters (`[A-Za-z]`), numbers (`[0-9]`),
|
|
1263
|
+
# underscores (`_`), or dashes (`-`). The maximum length of the entire ID
|
|
1261
1264
|
# is 1,024 characters. If `job_id` is provided, then `prefix` will not
|
|
1262
1265
|
# be used.
|
|
1263
1266
|
# @param [Hash] labels A hash of user-provided labels associated with
|
|
@@ -1481,7 +1484,7 @@ module Google
|
|
|
1481
1484
|
# use named query parameters. When set, `legacy_sql` will automatically be set to false and `standard_sql` to
|
|
1482
1485
|
# true.
|
|
1483
1486
|
#
|
|
1484
|
-
#
|
|
1487
|
+
# BigQuery types are converted from Ruby types as follows:
|
|
1485
1488
|
#
|
|
1486
1489
|
# | BigQuery | Ruby | Notes |
|
|
1487
1490
|
# |--------------|--------------------------------------|----------------------------------------------------|
|
|
@@ -1489,10 +1492,11 @@ module Google
|
|
|
1489
1492
|
# | `INT64` | `Integer` | |
|
|
1490
1493
|
# | `FLOAT64` | `Float` | |
|
|
1491
1494
|
# | `NUMERIC` | `BigDecimal` | `BigDecimal` values will be rounded to scale 9. |
|
|
1492
|
-
# | `BIGNUMERIC` |
|
|
1495
|
+
# | `BIGNUMERIC` | `BigDecimal` | NOT AUTOMATIC: Must be mapped using `types`, below.|
|
|
1493
1496
|
# | `STRING` | `String` | |
|
|
1494
1497
|
# | `DATETIME` | `DateTime` | `DATETIME` does not support time zone. |
|
|
1495
1498
|
# | `DATE` | `Date` | |
|
|
1499
|
+
# | `GEOGRAPHY` | `String` (WKT or GeoJSON) | NOT AUTOMATIC: Must be mapped using `types`, below.|
|
|
1496
1500
|
# | `TIMESTAMP` | `Time` | |
|
|
1497
1501
|
# | `TIME` | `Google::Cloud::BigQuery::Time` | |
|
|
1498
1502
|
# | `BYTES` | `File`, `IO`, `StringIO`, or similar | |
|
|
@@ -1500,7 +1504,8 @@ module Google
|
|
|
1500
1504
|
# | `STRUCT` | `Hash` | Hash keys may be strings or symbols. |
|
|
1501
1505
|
#
|
|
1502
1506
|
# See [Data Types](https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types) for an overview
|
|
1503
|
-
# of each BigQuery data type, including allowed values.
|
|
1507
|
+
# of each BigQuery data type, including allowed values. For the `GEOGRAPHY` type, see [Working with BigQuery
|
|
1508
|
+
# GIS data](https://cloud.google.com/bigquery/docs/gis-data).
|
|
1504
1509
|
# @param [Array, Hash] types Standard SQL only. Types of the SQL parameters in `params`. It is not always
|
|
1505
1510
|
# possible to infer the right SQL type from a value in `params`. In these cases, `types` must be used to
|
|
1506
1511
|
# specify the SQL type for these values.
|
|
@@ -1517,6 +1522,7 @@ module Google
|
|
|
1517
1522
|
# * `:STRING`
|
|
1518
1523
|
# * `:DATETIME`
|
|
1519
1524
|
# * `:DATE`
|
|
1525
|
+
# * `:GEOGRAPHY`
|
|
1520
1526
|
# * `:TIMESTAMP`
|
|
1521
1527
|
# * `:TIME`
|
|
1522
1528
|
# * `:BYTES`
|
|
@@ -1877,8 +1883,8 @@ module Google
|
|
|
1877
1883
|
# this option. Also note that for most use cases, the block yielded by
|
|
1878
1884
|
# this method is a more convenient way to configure the schema.
|
|
1879
1885
|
# @param [String] job_id A user-defined ID for the load job. The ID
|
|
1880
|
-
# must contain only letters (
|
|
1881
|
-
# (_), or dashes (
|
|
1886
|
+
# must contain only letters (`[A-Za-z]`), numbers (`[0-9]`), underscores
|
|
1887
|
+
# (`_`), or dashes (`-`). The maximum length is 1,024 characters. If
|
|
1882
1888
|
# `job_id` is provided, then `prefix` will not be used.
|
|
1883
1889
|
#
|
|
1884
1890
|
# See [Generating a job
|
|
@@ -1887,8 +1893,8 @@ module Google
|
|
|
1887
1893
|
# prepended to a generated value to produce a unique job ID. For
|
|
1888
1894
|
# example, the prefix `daily_import_job_` can be given to generate a
|
|
1889
1895
|
# job ID such as `daily_import_job_12vEDtMQ0mbp1Mo5Z7mzAFQJZazh`. The
|
|
1890
|
-
# prefix must contain only letters (
|
|
1891
|
-
# underscores (_), or dashes (
|
|
1896
|
+
# prefix must contain only letters (`[A-Za-z]`), numbers (`[0-9]`),
|
|
1897
|
+
# underscores (`_`), or dashes (`-`). The maximum length of the entire ID
|
|
1892
1898
|
# is 1,024 characters. If `job_id` is provided, then `prefix` will not
|
|
1893
1899
|
# be used.
|
|
1894
1900
|
# @param [Hash] labels A hash of user-provided labels associated with
|
|
@@ -2421,6 +2427,7 @@ module Google
|
|
|
2421
2427
|
# | `BIGNUMERIC` | `String` | Pass as `String` to avoid rounding to scale 9. |
|
|
2422
2428
|
# | `DATETIME` | `DateTime` | `DATETIME` does not support time zone. |
|
|
2423
2429
|
# | `DATE` | `Date` | |
|
|
2430
|
+
# | `GEOGRAPHY` | `String` | |
|
|
2424
2431
|
# | `TIMESTAMP` | `Time` | |
|
|
2425
2432
|
# | `TIME` | `Google::Cloud::BigQuery::Time` | |
|
|
2426
2433
|
# | `BYTES` | `File`, `IO`, `StringIO`, or similar | |
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# Copyright 2021 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
require "google/apis/bigquery_v2"
|
|
17
|
+
|
|
18
|
+
module Google
|
|
19
|
+
module Cloud
|
|
20
|
+
module Bigquery
|
|
21
|
+
module External
|
|
22
|
+
##
|
|
23
|
+
# # AvroSource
|
|
24
|
+
#
|
|
25
|
+
# {External::AvroSource} is a subclass of {External::DataSource} and
|
|
26
|
+
# represents a Avro external data source that can be queried
|
|
27
|
+
# from directly, even though the data is not stored in BigQuery. Instead
|
|
28
|
+
# of loading or streaming the data, this object references the external
|
|
29
|
+
# data source.
|
|
30
|
+
#
|
|
31
|
+
# @example
|
|
32
|
+
# require "google/cloud/bigquery"
|
|
33
|
+
#
|
|
34
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
35
|
+
#
|
|
36
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
|
37
|
+
# avro_table = bigquery.external avro_url do |avro|
|
|
38
|
+
# avro.use_avro_logical_types = 1
|
|
39
|
+
# end
|
|
40
|
+
#
|
|
41
|
+
# data = bigquery.query "SELECT * FROM my_ext_table",
|
|
42
|
+
# external: { my_ext_table: avro_table }
|
|
43
|
+
#
|
|
44
|
+
# # Iterate over the first page of results
|
|
45
|
+
# data.each do |row|
|
|
46
|
+
# puts row[:name]
|
|
47
|
+
# end
|
|
48
|
+
# # Retrieve the next page of results
|
|
49
|
+
# data = data.next if data.next?
|
|
50
|
+
#
|
|
51
|
+
class AvroSource < External::DataSource
|
|
52
|
+
##
|
|
53
|
+
# @private Create an empty AvroSource object.
|
|
54
|
+
def initialize
|
|
55
|
+
super
|
|
56
|
+
@gapi.avro_options = Google::Apis::BigqueryV2::AvroOptions.new
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
##
|
|
60
|
+
# Indicates whether to interpret logical types as the corresponding BigQuery data type (for example,
|
|
61
|
+
# `TIMESTAMP`), instead of using the raw type (for example, `INTEGER`).
|
|
62
|
+
#
|
|
63
|
+
# @return [Boolean]
|
|
64
|
+
#
|
|
65
|
+
# @example
|
|
66
|
+
# require "google/cloud/bigquery"
|
|
67
|
+
#
|
|
68
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
69
|
+
#
|
|
70
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
|
71
|
+
# avro_table = bigquery.external avro_url do |avro|
|
|
72
|
+
# avro.use_avro_logical_types = true
|
|
73
|
+
# end
|
|
74
|
+
#
|
|
75
|
+
# avro_table.use_avro_logical_types #=> true
|
|
76
|
+
#
|
|
77
|
+
def use_avro_logical_types
|
|
78
|
+
@gapi.avro_options.use_avro_logical_types
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
##
|
|
82
|
+
# Sets whether to interpret logical types as the corresponding BigQuery data type (for example, `TIMESTAMP`),
|
|
83
|
+
# instead of using the raw type (for example, `INTEGER`).
|
|
84
|
+
#
|
|
85
|
+
# @param [Boolean] new_use_avro_logical_types The new `use_avro_logical_types` value.
|
|
86
|
+
#
|
|
87
|
+
# @example
|
|
88
|
+
# require "google/cloud/bigquery"
|
|
89
|
+
#
|
|
90
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
91
|
+
#
|
|
92
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
|
93
|
+
# avro_table = bigquery.external avro_url do |avro|
|
|
94
|
+
# avro.use_avro_logical_types = true
|
|
95
|
+
# end
|
|
96
|
+
#
|
|
97
|
+
# avro_table.use_avro_logical_types #=> true
|
|
98
|
+
#
|
|
99
|
+
def use_avro_logical_types= new_use_avro_logical_types
|
|
100
|
+
frozen_check!
|
|
101
|
+
@gapi.avro_options.use_avro_logical_types = new_use_avro_logical_types
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
@@ -37,7 +37,7 @@ module Google
|
|
|
37
37
|
#
|
|
38
38
|
# bigquery = Google::Cloud::Bigquery.new
|
|
39
39
|
#
|
|
40
|
-
# avro_url = "gs://bucket/path/to
|
|
40
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
|
41
41
|
# avro_table = bigquery.external avro_url do |avro|
|
|
42
42
|
# avro.autodetect = true
|
|
43
43
|
# end
|
|
@@ -174,7 +174,7 @@ module Google
|
|
|
174
174
|
#
|
|
175
175
|
# bigquery = Google::Cloud::Bigquery.new
|
|
176
176
|
#
|
|
177
|
-
# avro_url = "gs://bucket/path/to
|
|
177
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
|
178
178
|
# avro_table = bigquery.external avro_url
|
|
179
179
|
#
|
|
180
180
|
# avro_table.format #=> "AVRO"
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
require "google/cloud/bigquery/external/data_source"
|
|
17
|
+
require "google/cloud/bigquery/external/avro_source"
|
|
17
18
|
require "google/cloud/bigquery/external/bigtable_source"
|
|
18
19
|
require "google/cloud/bigquery/external/csv_source"
|
|
19
20
|
require "google/cloud/bigquery/external/json_source"
|
|
@@ -101,28 +102,28 @@ module Google
|
|
|
101
102
|
# @private Determine source_format from inputs
|
|
102
103
|
def self.source_format_for urls, format
|
|
103
104
|
val = {
|
|
104
|
-
"csv" => "CSV",
|
|
105
105
|
"avro" => "AVRO",
|
|
106
|
-
"
|
|
107
|
-
"
|
|
108
|
-
"sheets" => "GOOGLE_SHEETS",
|
|
109
|
-
"google_sheets" => "GOOGLE_SHEETS",
|
|
110
|
-
"datastore" => "DATASTORE_BACKUP",
|
|
106
|
+
"bigtable" => "BIGTABLE",
|
|
107
|
+
"csv" => "CSV",
|
|
111
108
|
"backup" => "DATASTORE_BACKUP",
|
|
109
|
+
"datastore" => "DATASTORE_BACKUP",
|
|
112
110
|
"datastore_backup" => "DATASTORE_BACKUP",
|
|
113
|
-
"
|
|
111
|
+
"sheets" => "GOOGLE_SHEETS",
|
|
112
|
+
"google_sheets" => "GOOGLE_SHEETS",
|
|
113
|
+
"json" => "NEWLINE_DELIMITED_JSON",
|
|
114
|
+
"newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
|
|
114
115
|
"orc" => "ORC",
|
|
115
116
|
"parquet" => "PARQUET"
|
|
116
117
|
}[format.to_s.downcase]
|
|
117
118
|
return val unless val.nil?
|
|
118
119
|
Array(urls).each do |url|
|
|
119
|
-
return "CSV" if url.end_with? ".csv"
|
|
120
|
-
return "NEWLINE_DELIMITED_JSON" if url.end_with? ".json"
|
|
121
|
-
return "PARQUET" if url.end_with? ".parquet"
|
|
122
120
|
return "AVRO" if url.end_with? ".avro"
|
|
121
|
+
return "BIGTABLE" if url.start_with? "https://googleapis.com/bigtable/projects/"
|
|
122
|
+
return "CSV" if url.end_with? ".csv"
|
|
123
123
|
return "DATASTORE_BACKUP" if url.end_with? ".backup_info"
|
|
124
124
|
return "GOOGLE_SHEETS" if url.start_with? "https://docs.google.com/spreadsheets/"
|
|
125
|
-
return "
|
|
125
|
+
return "NEWLINE_DELIMITED_JSON" if url.end_with? ".json"
|
|
126
|
+
return "PARQUET" if url.end_with? ".parquet"
|
|
126
127
|
end
|
|
127
128
|
nil
|
|
128
129
|
end
|
|
@@ -131,13 +132,14 @@ module Google
|
|
|
131
132
|
# @private Determine table class from source_format
|
|
132
133
|
def self.table_class_for format
|
|
133
134
|
case format
|
|
135
|
+
when "AVRO" then External::AvroSource
|
|
136
|
+
when "BIGTABLE" then External::BigtableSource
|
|
134
137
|
when "CSV" then External::CsvSource
|
|
138
|
+
when "GOOGLE_SHEETS" then External::SheetsSource
|
|
135
139
|
when "NEWLINE_DELIMITED_JSON" then External::JsonSource
|
|
136
140
|
when "PARQUET" then External::ParquetSource
|
|
137
|
-
when "GOOGLE_SHEETS" then External::SheetsSource
|
|
138
|
-
when "BIGTABLE" then External::BigtableSource
|
|
139
141
|
else
|
|
140
|
-
#
|
|
142
|
+
# DATASTORE_BACKUP, ORC
|
|
141
143
|
External::DataSource
|
|
142
144
|
end
|
|
143
145
|
end
|
|
@@ -74,8 +74,8 @@ module Google
|
|
|
74
74
|
##
|
|
75
75
|
# The ID of the job.
|
|
76
76
|
#
|
|
77
|
-
# @return [String] The ID must contain only letters (
|
|
78
|
-
# (0-9), underscores (_), or dashes (
|
|
77
|
+
# @return [String] The ID must contain only letters (`[A-Za-z]`), numbers
|
|
78
|
+
# (`[0-9]`), underscores (`_`), or dashes (`-`). The maximum length is 1,024
|
|
79
79
|
# characters.
|
|
80
80
|
#
|
|
81
81
|
def job_id
|
|
@@ -226,6 +226,15 @@ module Google
|
|
|
226
226
|
Array(@gapi.statistics.reservation_usage).map { |g| ReservationUsage.from_gapi g }
|
|
227
227
|
end
|
|
228
228
|
|
|
229
|
+
##
|
|
230
|
+
# The ID of a multi-statement transaction.
|
|
231
|
+
#
|
|
232
|
+
# @return [String, nil] The transaction ID, or `nil` if not associated with a transaction.
|
|
233
|
+
#
|
|
234
|
+
def transaction_id
|
|
235
|
+
@gapi.statistics.transaction_info&.transaction_id
|
|
236
|
+
end
|
|
237
|
+
|
|
229
238
|
##
|
|
230
239
|
# The statistics including stack frames for a child job of a script.
|
|
231
240
|
#
|
|
@@ -383,6 +392,28 @@ module Google
|
|
|
383
392
|
true
|
|
384
393
|
end
|
|
385
394
|
|
|
395
|
+
##
|
|
396
|
+
# Requests that a job is deleted. This call will return when the job is deleted.
|
|
397
|
+
#
|
|
398
|
+
# @return [Boolean] Returns `true` if the job was deleted.
|
|
399
|
+
#
|
|
400
|
+
# @example
|
|
401
|
+
# require "google/cloud/bigquery"
|
|
402
|
+
#
|
|
403
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
404
|
+
#
|
|
405
|
+
# job = bigquery.job "my_job"
|
|
406
|
+
#
|
|
407
|
+
# job.delete
|
|
408
|
+
#
|
|
409
|
+
# @!group Lifecycle
|
|
410
|
+
#
|
|
411
|
+
def delete
|
|
412
|
+
ensure_service!
|
|
413
|
+
service.delete_job job_id, location: location
|
|
414
|
+
true
|
|
415
|
+
end
|
|
416
|
+
|
|
386
417
|
##
|
|
387
418
|
# Created a new job with the current configuration.
|
|
388
419
|
#
|
|
@@ -744,7 +744,7 @@ module Google
|
|
|
744
744
|
# See {Schema#string}.
|
|
745
745
|
#
|
|
746
746
|
# @param [String] name The field name. The name must contain only
|
|
747
|
-
# letters (
|
|
747
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
|
|
748
748
|
# start with a letter or underscore. The maximum length is 128
|
|
749
749
|
# characters.
|
|
750
750
|
# @param [String] description A description of the field.
|
|
@@ -778,7 +778,7 @@ module Google
|
|
|
778
778
|
# See {Schema#integer}.
|
|
779
779
|
#
|
|
780
780
|
# @param [String] name The field name. The name must contain only
|
|
781
|
-
# letters (
|
|
781
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
|
|
782
782
|
# start with a letter or underscore. The maximum length is 128
|
|
783
783
|
# characters.
|
|
784
784
|
# @param [String] description A description of the field.
|
|
@@ -810,7 +810,7 @@ module Google
|
|
|
810
810
|
# See {Schema#float}.
|
|
811
811
|
#
|
|
812
812
|
# @param [String] name The field name. The name must contain only
|
|
813
|
-
# letters (
|
|
813
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
|
|
814
814
|
# start with a letter or underscore. The maximum length is 128
|
|
815
815
|
# characters.
|
|
816
816
|
# @param [String] description A description of the field.
|
|
@@ -853,7 +853,7 @@ module Google
|
|
|
853
853
|
# See {Schema#numeric}
|
|
854
854
|
#
|
|
855
855
|
# @param [String] name The field name. The name must contain only
|
|
856
|
-
# letters (
|
|
856
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
|
|
857
857
|
# start with a letter or underscore. The maximum length is 128
|
|
858
858
|
# characters.
|
|
859
859
|
# @param [String] description A description of the field.
|
|
@@ -911,7 +911,7 @@ module Google
|
|
|
911
911
|
# See {Schema#bignumeric}
|
|
912
912
|
#
|
|
913
913
|
# @param [String] name The field name. The name must contain only
|
|
914
|
-
# letters (
|
|
914
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
|
|
915
915
|
# start with a letter or underscore. The maximum length is 128
|
|
916
916
|
# characters.
|
|
917
917
|
# @param [String] description A description of the field.
|
|
@@ -958,7 +958,7 @@ module Google
|
|
|
958
958
|
# See {Schema#boolean}.
|
|
959
959
|
#
|
|
960
960
|
# @param [String] name The field name. The name must contain only
|
|
961
|
-
# letters (
|
|
961
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
|
|
962
962
|
# start with a letter or underscore. The maximum length is 128
|
|
963
963
|
# characters.
|
|
964
964
|
# @param [String] description A description of the field.
|
|
@@ -990,7 +990,7 @@ module Google
|
|
|
990
990
|
# See {Schema#bytes}.
|
|
991
991
|
#
|
|
992
992
|
# @param [String] name The field name. The name must contain only
|
|
993
|
-
# letters (
|
|
993
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
|
|
994
994
|
# start with a letter or underscore. The maximum length is 128
|
|
995
995
|
# characters.
|
|
996
996
|
# @param [String] description A description of the field.
|
|
@@ -1024,7 +1024,7 @@ module Google
|
|
|
1024
1024
|
# See {Schema#timestamp}.
|
|
1025
1025
|
#
|
|
1026
1026
|
# @param [String] name The field name. The name must contain only
|
|
1027
|
-
# letters (
|
|
1027
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
|
|
1028
1028
|
# start with a letter or underscore. The maximum length is 128
|
|
1029
1029
|
# characters.
|
|
1030
1030
|
# @param [String] description A description of the field.
|
|
@@ -1056,7 +1056,7 @@ module Google
|
|
|
1056
1056
|
# See {Schema#time}.
|
|
1057
1057
|
#
|
|
1058
1058
|
# @param [String] name The field name. The name must contain only
|
|
1059
|
-
# letters (
|
|
1059
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
|
|
1060
1060
|
# start with a letter or underscore. The maximum length is 128
|
|
1061
1061
|
# characters.
|
|
1062
1062
|
# @param [String] description A description of the field.
|
|
@@ -1088,7 +1088,7 @@ module Google
|
|
|
1088
1088
|
# See {Schema#datetime}.
|
|
1089
1089
|
#
|
|
1090
1090
|
# @param [String] name The field name. The name must contain only
|
|
1091
|
-
# letters (
|
|
1091
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
|
|
1092
1092
|
# start with a letter or underscore. The maximum length is 128
|
|
1093
1093
|
# characters.
|
|
1094
1094
|
# @param [String] description A description of the field.
|
|
@@ -1120,7 +1120,7 @@ module Google
|
|
|
1120
1120
|
# See {Schema#date}.
|
|
1121
1121
|
#
|
|
1122
1122
|
# @param [String] name The field name. The name must contain only
|
|
1123
|
-
# letters (
|
|
1123
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
|
|
1124
1124
|
# start with a letter or underscore. The maximum length is 128
|
|
1125
1125
|
# characters.
|
|
1126
1126
|
# @param [String] description A description of the field.
|
|
@@ -1146,6 +1146,42 @@ module Google
|
|
|
1146
1146
|
schema.date name, description: description, mode: mode, policy_tags: policy_tags
|
|
1147
1147
|
end
|
|
1148
1148
|
|
|
1149
|
+
##
|
|
1150
|
+
# Adds a geography field to the schema.
|
|
1151
|
+
#
|
|
1152
|
+
# See {Schema#geography}.
|
|
1153
|
+
#
|
|
1154
|
+
# @see https://cloud.google.com/bigquery/docs/gis-data Working with BigQuery GIS data
|
|
1155
|
+
#
|
|
1156
|
+
# @param [String] name The field name. The name must contain only
|
|
1157
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
|
|
1158
|
+
# start with a letter or underscore. The maximum length is 128
|
|
1159
|
+
# characters.
|
|
1160
|
+
# @param [String] description A description of the field.
|
|
1161
|
+
# @param [Symbol] mode The field's mode. The possible values are
|
|
1162
|
+
# `:nullable`, `:required`, and `:repeated`. The default value is
|
|
1163
|
+
# `:nullable`.
|
|
1164
|
+
# @param [Array<String>, String] policy_tags The policy tag list or
|
|
1165
|
+
# single policy tag for the field. Policy tag identifiers are of
|
|
1166
|
+
# the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
|
|
1167
|
+
# At most 1 policy tag is currently allowed.
|
|
1168
|
+
#
|
|
1169
|
+
# @example
|
|
1170
|
+
# require "google/cloud/bigquery"
|
|
1171
|
+
#
|
|
1172
|
+
# bigquery = Google::Cloud::Bigquery.new
|
|
1173
|
+
# dataset = bigquery.dataset "my_dataset"
|
|
1174
|
+
# job = dataset.load_job "my_table", "gs://abc/file" do |schema|
|
|
1175
|
+
# schema.record "cities_lived", mode: :repeated do |cities_lived|
|
|
1176
|
+
# cities_lived.geography "location", mode: :required
|
|
1177
|
+
# cities_lived.integer "number_of_years", mode: :required
|
|
1178
|
+
# end
|
|
1179
|
+
# end
|
|
1180
|
+
#
|
|
1181
|
+
def geography name, description: nil, mode: :nullable, policy_tags: nil
|
|
1182
|
+
schema.geography name, description: description, mode: mode, policy_tags: policy_tags
|
|
1183
|
+
end
|
|
1184
|
+
|
|
1149
1185
|
##
|
|
1150
1186
|
# Adds a record field to the schema. A block must be passed describing
|
|
1151
1187
|
# the nested fields of the record. For more information about nested
|
|
@@ -1156,7 +1192,7 @@ module Google
|
|
|
1156
1192
|
# See {Schema#record}.
|
|
1157
1193
|
#
|
|
1158
1194
|
# @param [String] name The field name. The name must contain only
|
|
1159
|
-
# letters (
|
|
1195
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
|
|
1160
1196
|
# start with a letter or underscore. The maximum length is 128
|
|
1161
1197
|
# characters.
|
|
1162
1198
|
# @param [String] description A description of the field.
|