google-cloud-bigquery 1.59.0 → 1.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AUTHENTICATION.md +12 -1
- data/CHANGELOG.md +15 -0
- data/lib/google/cloud/bigquery/copy_job.rb +14 -1
- data/lib/google/cloud/bigquery/dataset.rb +241 -18
- data/lib/google/cloud/bigquery/extract_job.rb +14 -1
- data/lib/google/cloud/bigquery/load_job.rb +14 -2
- data/lib/google/cloud/bigquery/model.rb +15 -4
- data/lib/google/cloud/bigquery/project.rb +113 -18
- data/lib/google/cloud/bigquery/query_job.rb +14 -1
- data/lib/google/cloud/bigquery/service.rb +8 -6
- data/lib/google/cloud/bigquery/table.rb +115 -22
- data/lib/google/cloud/bigquery/version.rb +1 -1
- data/lib/google/cloud/bigquery.rb +25 -9
- data/lib/google-cloud-bigquery.rb +19 -3
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ae087fd9a1d2cc0921fe7b72c505fe3828add1beefa9482f4323de8759a6ef1f
|
|
4
|
+
data.tar.gz: 7390b66a15f2b1b63efd8ce65b94190f437d8f20a3184512722e41d5f9a80e3f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d645597d965f183c9eefcdbbd7d5759eef90f05a68ac5210291dab1146924bb0bc3bd0b5c25f42055b4750474125123333333de177e40fb100d160af179eee62
|
|
7
|
+
data.tar.gz: c08b54e15f8fdc7177f97ee7d073ab32057bce243ce0465dc26c121a29cbf95f913f1045eaafffcf614037e13c47d37b58dd00d386f30f268def6a50425b73b5
|
data/AUTHENTICATION.md
CHANGED
|
@@ -28,6 +28,12 @@ providing **Project ID** and **Service Account Credentials** directly in code.
|
|
|
28
28
|
|
|
29
29
|
**Credentials** are discovered in the following order:
|
|
30
30
|
|
|
31
|
+
> [!WARNING]
|
|
32
|
+
> If you accept a credential configuration (JSON file or Hash) from an
|
|
33
|
+
> external source for authentication to Google Cloud, you must validate it before
|
|
34
|
+
> providing it to a Google API client library. Providing an unvalidated credential
|
|
35
|
+
> configuration to Google APIs can compromise the security of your systems and data.
|
|
36
|
+
|
|
31
37
|
1. Specify credentials in method arguments
|
|
32
38
|
2. Specify credentials in configuration
|
|
33
39
|
3. Discover credentials path in environment variables
|
|
@@ -81,11 +87,16 @@ The **Project ID** and the path to the **Credentials JSON** file can be configur
|
|
|
81
87
|
instead of placing them in environment variables or providing them as arguments.
|
|
82
88
|
|
|
83
89
|
```ruby
|
|
90
|
+
require "googleauth"
|
|
84
91
|
require "google/cloud/bigquery"
|
|
85
92
|
|
|
93
|
+
credentials = ::Google::Auth::ServiceAccountCredentials.make_creds(
|
|
94
|
+
json_key_io: ::File.open("/path/to/keyfile.json")
|
|
95
|
+
)
|
|
96
|
+
|
|
86
97
|
Google::Cloud::Bigquery.configure do |config|
|
|
87
98
|
config.project_id = "my-project-id"
|
|
88
|
-
config.credentials =
|
|
99
|
+
config.credentials = credentials
|
|
89
100
|
end
|
|
90
101
|
|
|
91
102
|
bigquery = Google::Cloud::Bigquery.new
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
# Release History
|
|
2
2
|
|
|
3
|
+
### 1.61.0 (2025-11-04)
|
|
4
|
+
|
|
5
|
+
#### Features
|
|
6
|
+
|
|
7
|
+
* Support fine grained ACLs ([#31772](https://github.com/googleapis/google-cloud-ruby/issues/31772))
|
|
8
|
+
#### Documentation
|
|
9
|
+
|
|
10
|
+
* add warning about loading unvalidated credentials ([#32121](https://github.com/googleapis/google-cloud-ruby/issues/32121))
|
|
11
|
+
|
|
12
|
+
### 1.60.0 (2025-10-24)
|
|
13
|
+
|
|
14
|
+
#### Features
|
|
15
|
+
|
|
16
|
+
* Exposing reservation field in JobConfiguration ([#31777](https://github.com/googleapis/google-cloud-ruby/issues/31777))
|
|
17
|
+
|
|
3
18
|
### 1.59.0 (2025-09-17)
|
|
4
19
|
|
|
5
20
|
#### Features
|
|
@@ -194,7 +194,8 @@ module Google
|
|
|
194
194
|
job_reference: job_ref,
|
|
195
195
|
configuration: Google::Apis::BigqueryV2::JobConfiguration.new(
|
|
196
196
|
copy: copy_cfg,
|
|
197
|
-
dry_run: options[:dryrun]
|
|
197
|
+
dry_run: options[:dryrun],
|
|
198
|
+
reservation: options[:reservation]
|
|
198
199
|
)
|
|
199
200
|
)
|
|
200
201
|
|
|
@@ -325,6 +326,18 @@ module Google
|
|
|
325
326
|
@gapi.configuration.update! labels: value
|
|
326
327
|
end
|
|
327
328
|
|
|
329
|
+
##
|
|
330
|
+
# Sets the reservation that job would use. User can specify a reservation
|
|
331
|
+
# to execute the job. If reservation is not set, reservation is determined
|
|
332
|
+
# based on the rules defined by the reservation assignments. The expected
|
|
333
|
+
# format is `projects/`project`/locations/`location`/reservations/`reservation``.
|
|
334
|
+
# @param [String] value The reservation name.
|
|
335
|
+
#
|
|
336
|
+
# @!group Attributes
|
|
337
|
+
def reservation= value
|
|
338
|
+
@gapi.configuration.update! reservation: value
|
|
339
|
+
end
|
|
340
|
+
|
|
328
341
|
def cancel
|
|
329
342
|
raise "not implemented in #{self.class}"
|
|
330
343
|
end
|
|
@@ -29,6 +29,48 @@ require "google/apis/bigquery_v2"
|
|
|
29
29
|
module Google
|
|
30
30
|
module Cloud
|
|
31
31
|
module Bigquery
|
|
32
|
+
module DatasetView
|
|
33
|
+
# Provides constants for the dataset_view parameter, an optional field
|
|
34
|
+
# in the GetDatasetRequest used to specify which information about a
|
|
35
|
+
# BigQuery dataset should be returned in the response. By controlling
|
|
36
|
+
# this parameter, users can request a partial or full response, which
|
|
37
|
+
# helps enforce fine-grained access control based on their permissions.
|
|
38
|
+
|
|
39
|
+
# Default. Equivalent to `FULL`. `datasets.get` and `datasets.getIamPolicy` permissions required.
|
|
40
|
+
DATASET_VIEW_UNSPECIFIED = "DATASET_VIEW_UNSPECIFIED".freeze
|
|
41
|
+
|
|
42
|
+
# Returns metadata only. `datasets.get` permission required.
|
|
43
|
+
METADATA = "METADATA".freeze
|
|
44
|
+
|
|
45
|
+
# Returns ACLs only. `datasets.getIamPolicy` permission required.
|
|
46
|
+
ACL = "ACL".freeze
|
|
47
|
+
|
|
48
|
+
# Returns metadata and ACLs. `datasets.get` and `datasets.getIamPolicy` permissions required.
|
|
49
|
+
FULL = "FULL".freeze
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
module UpdateMode
|
|
53
|
+
# Provides constants for the update_mode parameter, an optional field
|
|
54
|
+
# in the PatchDatasetRequest and UpdateDatasetRequest used to specify
|
|
55
|
+
# whether the resource is being updated with full or partial semantics
|
|
56
|
+
# (metadata, ACLs, or both).
|
|
57
|
+
# By controlling this parameter, users can request full or partial
|
|
58
|
+
# update semantics, which helps enforce fine-grained access control
|
|
59
|
+
# based on their permissions.
|
|
60
|
+
|
|
61
|
+
# Default. Equivalent to `UPDATE_FULL`. `datasets.update` and `datasets.setIamPolicy` permissions required.
|
|
62
|
+
UPDATE_MODE_UNSPECIFIED = "UPDATE_MODE_UNSPECIFIED".freeze
|
|
63
|
+
|
|
64
|
+
# Updates both metadata and ACLs. `datasets.update` and `datasets.setIamPolicy` permissions required.
|
|
65
|
+
UPDATE_FULL = "UPDATE_FULL".freeze
|
|
66
|
+
|
|
67
|
+
# Updates only metadata. `datasets.update` permission required.
|
|
68
|
+
UPDATE_METADATA = "UPDATE_METADATA".freeze
|
|
69
|
+
|
|
70
|
+
# Updates only ACLs. `datasets.setIamPolicy` permission required.
|
|
71
|
+
UPDATE_ACL = "UPDATE_ACL".freeze
|
|
72
|
+
end
|
|
73
|
+
|
|
32
74
|
##
|
|
33
75
|
# # Dataset
|
|
34
76
|
#
|
|
@@ -63,6 +105,13 @@ module Google
|
|
|
63
105
|
# @private Access Policy Version for get, update, patch, and insert API calls
|
|
64
106
|
attr_accessor :access_policy_version
|
|
65
107
|
|
|
108
|
+
##
|
|
109
|
+
# @private The dataset_view parameter is an optional field in the GetDatasetRequest used to specify which
|
|
110
|
+
# information about a BigQuery dataset should be returned in the response. By controlling this parameter, users
|
|
111
|
+
# can request a partial or full response, which helps enforce fine-grained access control based on their
|
|
112
|
+
# permissions.
|
|
113
|
+
attr_accessor :dataset_view
|
|
114
|
+
|
|
66
115
|
##
|
|
67
116
|
# @private Create an empty Dataset object.
|
|
68
117
|
def initialize
|
|
@@ -70,6 +119,7 @@ module Google
|
|
|
70
119
|
@gapi = nil
|
|
71
120
|
@reference = nil
|
|
72
121
|
@access_policy_version = nil
|
|
122
|
+
@dataset_view = nil
|
|
73
123
|
end
|
|
74
124
|
|
|
75
125
|
##
|
|
@@ -1424,6 +1474,12 @@ module Google
|
|
|
1424
1474
|
# The default value is false.
|
|
1425
1475
|
# @param [String] session_id The ID of an existing session. See also the
|
|
1426
1476
|
# `create_session` param and {Job#session_id}.
|
|
1477
|
+
# @param [String] reservation The reservation that job would use. User
|
|
1478
|
+
# can specify a reservation to execute the job. If reservation is not
|
|
1479
|
+
# set, reservation is determined based on the rules defined by the
|
|
1480
|
+
# reservation assignments. The expected format is
|
|
1481
|
+
# `projects/`project`/locations/`location`/reservations/`reservation``.
|
|
1482
|
+
#
|
|
1427
1483
|
# @yield [job] a job configuration object
|
|
1428
1484
|
# @yieldparam [Google::Cloud::Bigquery::QueryJob::Updater] job a job
|
|
1429
1485
|
# configuration object for setting additional options for the query.
|
|
@@ -1596,7 +1652,8 @@ module Google
|
|
|
1596
1652
|
labels: nil,
|
|
1597
1653
|
udfs: nil,
|
|
1598
1654
|
create_session: nil,
|
|
1599
|
-
session_id: nil
|
|
1655
|
+
session_id: nil,
|
|
1656
|
+
reservation: nil
|
|
1600
1657
|
ensure_service!
|
|
1601
1658
|
options = {
|
|
1602
1659
|
params: params,
|
|
@@ -1619,7 +1676,8 @@ module Google
|
|
|
1619
1676
|
labels: labels,
|
|
1620
1677
|
udfs: udfs,
|
|
1621
1678
|
create_session: create_session,
|
|
1622
|
-
session_id: session_id
|
|
1679
|
+
session_id: session_id,
|
|
1680
|
+
reservation: reservation
|
|
1623
1681
|
}
|
|
1624
1682
|
|
|
1625
1683
|
updater = QueryJob::Updater.from_options service, query, options
|
|
@@ -1748,6 +1806,12 @@ module Google
|
|
|
1748
1806
|
# `create_session` param in {#query_job} and {Job#session_id}.
|
|
1749
1807
|
# @param [Boolean] format_options_use_int64_timestamp Output timestamp
|
|
1750
1808
|
# as usec int64. Default is true.
|
|
1809
|
+
# @param [String] reservation The reservation that job would use. User
|
|
1810
|
+
# can specify a reservation to execute the job. If reservation is not
|
|
1811
|
+
# set, reservation is determined based on the rules defined by the
|
|
1812
|
+
# reservation assignments. The expected format is
|
|
1813
|
+
# `projects/`project`/locations/`location`/reservations/`reservation``.
|
|
1814
|
+
#
|
|
1751
1815
|
# @yield [job] a job configuration object
|
|
1752
1816
|
# @yieldparam [Google::Cloud::Bigquery::QueryJob::Updater] job a job
|
|
1753
1817
|
# configuration object for setting additional options for the query.
|
|
@@ -1903,6 +1967,7 @@ module Google
|
|
|
1903
1967
|
legacy_sql: nil,
|
|
1904
1968
|
session_id: nil,
|
|
1905
1969
|
format_options_use_int64_timestamp: true,
|
|
1970
|
+
reservation: nil,
|
|
1906
1971
|
&block
|
|
1907
1972
|
job = query_job query,
|
|
1908
1973
|
params: params,
|
|
@@ -1912,6 +1977,7 @@ module Google
|
|
|
1912
1977
|
standard_sql: standard_sql,
|
|
1913
1978
|
legacy_sql: legacy_sql,
|
|
1914
1979
|
session_id: session_id,
|
|
1980
|
+
reservation: reservation,
|
|
1915
1981
|
&block
|
|
1916
1982
|
job.wait_until_done!
|
|
1917
1983
|
ensure_job_succeeded! job
|
|
@@ -2130,6 +2196,53 @@ module Google
|
|
|
2130
2196
|
# and the load job will happen in the table created within that session.
|
|
2131
2197
|
# Note: This will work only for _SESSION dataset.
|
|
2132
2198
|
# @param [string] session_id Session ID in which the load job must run.
|
|
2199
|
+
# @param [String] date_format Format used to parse DATE values.
|
|
2200
|
+
# Supports SQL-style format strings. See
|
|
2201
|
+
# [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
|
|
2202
|
+
# @param [String] datetime_format Format used to parse DATETIME
|
|
2203
|
+
# values. Supports SQL-style format strings. See
|
|
2204
|
+
# [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
|
|
2205
|
+
# @param [String] time_format Format used to parse TIME values.
|
|
2206
|
+
# Supports SQL-style format strings. See
|
|
2207
|
+
# [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
|
|
2208
|
+
# @param [String] timestamp_format Format used to parse
|
|
2209
|
+
# TIMESTAMP values. Supports SQL-style format strings. See
|
|
2210
|
+
# [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
|
|
2211
|
+
# @param [Array<String>] null_markers A list of strings represented as
|
|
2212
|
+
# SQL NULL value in a CSV file. null_marker and null_markers can't be
|
|
2213
|
+
# set at the same time. If null_marker is set, null_markers has to be
|
|
2214
|
+
# not set. If null_markers is set, null_marker has to be not set. If
|
|
2215
|
+
# both null_marker and null_markers are set at the same time, a user
|
|
2216
|
+
# error would be thrown. Any strings listed in null_markers, including
|
|
2217
|
+
# empty string would be interpreted as SQL NULL. This applies to all
|
|
2218
|
+
# column types.
|
|
2219
|
+
# @param [String] source_column_match Controls the strategy used to
|
|
2220
|
+
# match loaded columns to the schema. If not set, a sensible default is
|
|
2221
|
+
# chosen based on how the schema is provided. If autodetect is used,
|
|
2222
|
+
# then columns are matched by name. Otherwise, columns are matched by
|
|
2223
|
+
# position. This is done to keep the behavior backward-compatible.
|
|
2224
|
+
#
|
|
2225
|
+
# Acceptable values are:
|
|
2226
|
+
# * `POSITION` - matches by position. This assumes that the columns are
|
|
2227
|
+
# ordered the same way as the schema.
|
|
2228
|
+
# * `NAME` - matches by name. This reads the header row as column names
|
|
2229
|
+
# and reorders columns to match the field names in the schema.
|
|
2230
|
+
# @param [String] time_zone The time zone used when parsing timestamp
|
|
2231
|
+
# values.
|
|
2232
|
+
# @param [String] reference_file_schema_uri The URI of the reference
|
|
2233
|
+
# file with the reader schema. This file is only loaded if it is part
|
|
2234
|
+
# of source URIs, but is not loaded otherwise. It is enabled for the
|
|
2235
|
+
# following formats: `AVRO`, `PARQUET`, `ORC`.
|
|
2236
|
+
# @param [Boolean] preserve_ascii_control_characters When source_format
|
|
2237
|
+
# is set to `CSV`, indicates if the embedded ASCII control characters
|
|
2238
|
+
# (the first 32 characters in the ASCII-table, from `\x00` to `\x1F`)
|
|
2239
|
+
# are preserved. By default, ASCII control characters are not
|
|
2240
|
+
# preserved.
|
|
2241
|
+
# @param [String] reservation The reservation that job would use. User
|
|
2242
|
+
# can specify a reservation to execute the job. If reservation is not
|
|
2243
|
+
# set, reservation is determined based on the rules defined by the
|
|
2244
|
+
# reservation assignments. The expected format is
|
|
2245
|
+
# `projects/`project`/locations/`location`/reservations/`reservation``.
|
|
2133
2246
|
#
|
|
2134
2247
|
# @yield [updater] A block for setting the schema and other
|
|
2135
2248
|
# options for the destination table. The schema can be omitted if the
|
|
@@ -2224,7 +2337,10 @@ module Google
|
|
|
2224
2337
|
def load_job table_id, files, format: nil, create: nil, write: nil, projection_fields: nil, jagged_rows: nil,
|
|
2225
2338
|
quoted_newlines: nil, encoding: nil, delimiter: nil, ignore_unknown: nil, max_bad_records: nil,
|
|
2226
2339
|
quote: nil, skip_leading: nil, schema: nil, job_id: nil, prefix: nil, labels: nil, autodetect: nil,
|
|
2227
|
-
null_marker: nil, dryrun: nil, create_session: nil, session_id: nil
|
|
2340
|
+
null_marker: nil, dryrun: nil, create_session: nil, session_id: nil, date_format: nil,
|
|
2341
|
+
datetime_format: nil, time_format: nil, timestamp_format: nil, null_markers: nil,
|
|
2342
|
+
source_column_match: nil, time_zone: nil, reference_file_schema_uri: nil,
|
|
2343
|
+
preserve_ascii_control_characters: nil, reservation: nil
|
|
2228
2344
|
ensure_service!
|
|
2229
2345
|
|
|
2230
2346
|
updater = load_job_updater table_id,
|
|
@@ -2234,7 +2350,13 @@ module Google
|
|
|
2234
2350
|
max_bad_records: max_bad_records, quote: quote, skip_leading: skip_leading,
|
|
2235
2351
|
dryrun: dryrun, schema: schema, job_id: job_id, prefix: prefix, labels: labels,
|
|
2236
2352
|
autodetect: autodetect, null_marker: null_marker, create_session: create_session,
|
|
2237
|
-
session_id: session_id
|
|
2353
|
+
session_id: session_id, date_format: date_format, datetime_format: datetime_format,
|
|
2354
|
+
time_format: time_format, timestamp_format: timestamp_format,
|
|
2355
|
+
null_markers: null_markers, source_column_match: source_column_match,
|
|
2356
|
+
time_zone: time_zone, reference_file_schema_uri: reference_file_schema_uri,
|
|
2357
|
+
preserve_ascii_control_characters: preserve_ascii_control_characters,
|
|
2358
|
+
reservation: reservation
|
|
2359
|
+
|
|
2238
2360
|
|
|
2239
2361
|
yield updater if block_given?
|
|
2240
2362
|
|
|
@@ -2361,7 +2483,53 @@ module Google
|
|
|
2361
2483
|
# this option. Also note that for most use cases, the block yielded by
|
|
2362
2484
|
# this method is a more convenient way to configure the schema.
|
|
2363
2485
|
# @param [string] session_id Session ID in which the load job must run.
|
|
2364
|
-
#
|
|
2486
|
+
# @param [String] date_format Format used to parse DATE values.
|
|
2487
|
+
# Supports SQL-style format strings. See
|
|
2488
|
+
# [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
|
|
2489
|
+
# @param [String] datetime_format Format used to parse DATETIME
|
|
2490
|
+
# values. Supports SQL-style format strings. See
|
|
2491
|
+
# [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
|
|
2492
|
+
# @param [String] time_format Format used to parse TIME values.
|
|
2493
|
+
# Supports SQL-style format strings. See
|
|
2494
|
+
# [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
|
|
2495
|
+
# @param [String] timestamp_format Format used to parse
|
|
2496
|
+
# TIMESTAMP values. Supports SQL-style format strings. See
|
|
2497
|
+
# [date and time formatting guide](https://cloud.google.com/bigquery/docs/reference/standard-sql/format-elements#format_date_time_as_string)
|
|
2498
|
+
# @param [Array<String>] null_markers A list of strings represented as
|
|
2499
|
+
# SQL NULL value in a CSV file. null_marker and null_markers can't be
|
|
2500
|
+
# set at the same time. If null_marker is set, null_markers has to be
|
|
2501
|
+
# not set. If null_markers is set, null_marker has to be not set. If
|
|
2502
|
+
# both null_marker and null_markers are set at the same time, a user
|
|
2503
|
+
# error would be thrown. Any strings listed in null_markers, including
|
|
2504
|
+
# empty string would be interpreted as SQL NULL. This applies to all
|
|
2505
|
+
# column types.
|
|
2506
|
+
# @param [String] source_column_match Controls the strategy used to
|
|
2507
|
+
# match loaded columns to the schema. If not set, a sensible default is
|
|
2508
|
+
# chosen based on how the schema is provided. If autodetect is used,
|
|
2509
|
+
# then columns are matched by name. Otherwise, columns are matched by
|
|
2510
|
+
# position. This is done to keep the behavior backward-compatible.
|
|
2511
|
+
#
|
|
2512
|
+
# Acceptable values are:
|
|
2513
|
+
# * `POSITION` - matches by position. This assumes that the columns are
|
|
2514
|
+
# ordered the same way as the schema.
|
|
2515
|
+
# * `NAME` - matches by name. This reads the header row as column names
|
|
2516
|
+
# and reorders columns to match the field names in the schema.
|
|
2517
|
+
# @param [String] time_zone The time zone used when parsing timestamp
|
|
2518
|
+
# values.
|
|
2519
|
+
# @param [String] reference_file_schema_uri The URI of the reference
|
|
2520
|
+
# file with the reader schema. This file is only loaded if it is part
|
|
2521
|
+
# of source URIs, but is not loaded otherwise. It is enabled for the
|
|
2522
|
+
# following formats: `AVRO`, `PARQUET`, `ORC`.
|
|
2523
|
+
# @param [Boolean] preserve_ascii_control_characters When source_format
|
|
2524
|
+
# is set to `CSV`, indicates if the embedded ASCII control characters
|
|
2525
|
+
# (the first 32 characters in the ASCII-table, from `\x00` to `\x1F`)
|
|
2526
|
+
# are preserved. By default, ASCII control characters are not
|
|
2527
|
+
# preserved.
|
|
2528
|
+
# @param [String] reservation The reservation that job would use. User
|
|
2529
|
+
# can specify a reservation to execute the job. If reservation is not
|
|
2530
|
+
# set, reservation is determined based on the rules defined by the
|
|
2531
|
+
# reservation assignments. The expected format is
|
|
2532
|
+
# `projects/`project`/locations/`location`/reservations/`reservation``.
|
|
2365
2533
|
#
|
|
2366
2534
|
# @yield [updater] A block for setting the schema of the destination
|
|
2367
2535
|
# table and other options for the load job. The schema can be omitted
|
|
@@ -2454,13 +2622,22 @@ module Google
|
|
|
2454
2622
|
#
|
|
2455
2623
|
def load table_id, files, format: nil, create: nil, write: nil, projection_fields: nil, jagged_rows: nil,
|
|
2456
2624
|
quoted_newlines: nil, encoding: nil, delimiter: nil, ignore_unknown: nil, max_bad_records: nil,
|
|
2457
|
-
quote: nil, skip_leading: nil, schema: nil, autodetect: nil, null_marker: nil, session_id: nil,
|
|
2625
|
+
quote: nil, skip_leading: nil, schema: nil, autodetect: nil, null_marker: nil, session_id: nil,
|
|
2626
|
+
date_format: nil, datetime_format: nil, time_format: nil, timestamp_format: nil,
|
|
2627
|
+
null_markers: nil, source_column_match: nil, time_zone: nil, reference_file_schema_uri: nil,
|
|
2628
|
+
preserve_ascii_control_characters: nil,
|
|
2629
|
+
reservation: nil, &block
|
|
2458
2630
|
job = load_job table_id, files,
|
|
2459
2631
|
format: format, create: create, write: write, projection_fields: projection_fields,
|
|
2460
2632
|
jagged_rows: jagged_rows, quoted_newlines: quoted_newlines, encoding: encoding,
|
|
2461
2633
|
delimiter: delimiter, ignore_unknown: ignore_unknown, max_bad_records: max_bad_records,
|
|
2462
2634
|
quote: quote, skip_leading: skip_leading, schema: schema, autodetect: autodetect,
|
|
2463
|
-
null_marker: null_marker, session_id: session_id,
|
|
2635
|
+
null_marker: null_marker, session_id: session_id, date_format: date_format,
|
|
2636
|
+
datetime_format: datetime_format, time_format: time_format, timestamp_format: timestamp_format,
|
|
2637
|
+
null_markers: null_markers, source_column_match: source_column_match, time_zone: time_zone,
|
|
2638
|
+
reference_file_schema_uri: reference_file_schema_uri,
|
|
2639
|
+
preserve_ascii_control_characters: preserve_ascii_control_characters,
|
|
2640
|
+
reservation: reservation, &block
|
|
2464
2641
|
|
|
2465
2642
|
job.wait_until_done!
|
|
2466
2643
|
ensure_job_succeeded! job
|
|
@@ -2483,7 +2660,8 @@ module Google
|
|
|
2483
2660
|
#
|
|
2484
2661
|
def reload!
|
|
2485
2662
|
ensure_service!
|
|
2486
|
-
@gapi = service.get_project_dataset project_id, dataset_id, access_policy_version: @access_policy_version
|
|
2663
|
+
@gapi = service.get_project_dataset project_id, dataset_id, access_policy_version: @access_policy_version,
|
|
2664
|
+
dataset_view: @dataset_view
|
|
2487
2665
|
@reference = nil
|
|
2488
2666
|
@exists = nil
|
|
2489
2667
|
self
|
|
@@ -2612,11 +2790,12 @@ module Google
|
|
|
2612
2790
|
|
|
2613
2791
|
##
|
|
2614
2792
|
# @private New Dataset from a Google API Client object.
|
|
2615
|
-
def self.from_gapi gapi, conn, access_policy_version: nil
|
|
2793
|
+
def self.from_gapi gapi, conn, access_policy_version: nil, dataset_view: nil
|
|
2616
2794
|
new.tap do |f|
|
|
2617
2795
|
f.gapi = gapi
|
|
2618
2796
|
f.service = conn
|
|
2619
2797
|
f.access_policy_version = access_policy_version
|
|
2798
|
+
f.dataset_view = dataset_view
|
|
2620
2799
|
end
|
|
2621
2800
|
end
|
|
2622
2801
|
|
|
@@ -2937,10 +3116,25 @@ module Google
|
|
|
2937
3116
|
def patch_gapi! *attributes
|
|
2938
3117
|
return if attributes.empty?
|
|
2939
3118
|
ensure_service!
|
|
3119
|
+
|
|
2940
3120
|
patch_args = attributes.to_h { |attr| [attr, @gapi.send(attr)] }
|
|
3121
|
+
|
|
3122
|
+
update_mode = nil
|
|
3123
|
+
has_access_key = patch_args.key? :access
|
|
3124
|
+
other_keys_exist = (patch_args.keys - [:access]).any?
|
|
3125
|
+
|
|
3126
|
+
if has_access_key && !other_keys_exist
|
|
3127
|
+
update_mode = UpdateMode::UPDATE_ACL
|
|
3128
|
+
elsif !has_access_key && other_keys_exist
|
|
3129
|
+
update_mode = UpdateMode::UPDATE_METADATA
|
|
3130
|
+
elsif has_access_key && other_keys_exist
|
|
3131
|
+
update_mode = UpdateMode::FULL
|
|
3132
|
+
end
|
|
3133
|
+
|
|
2941
3134
|
patch_gapi = Google::Apis::BigqueryV2::Dataset.new(**patch_args)
|
|
2942
3135
|
patch_gapi.etag = etag if etag
|
|
2943
|
-
@gapi = service.patch_dataset dataset_id, patch_gapi, access_policy_version: @access_policy_version
|
|
3136
|
+
@gapi = service.patch_dataset dataset_id, patch_gapi, access_policy_version: @access_policy_version,
|
|
3137
|
+
update_mode: update_mode
|
|
2944
3138
|
end
|
|
2945
3139
|
|
|
2946
3140
|
##
|
|
@@ -2961,7 +3155,7 @@ module Google
|
|
|
2961
3155
|
end
|
|
2962
3156
|
end
|
|
2963
3157
|
|
|
2964
|
-
def load_job_gapi table_id, dryrun, job_id: nil, prefix: nil
|
|
3158
|
+
def load_job_gapi table_id, dryrun, job_id: nil, prefix: nil, reservation: nil
|
|
2965
3159
|
job_ref = service.job_ref_from job_id, prefix
|
|
2966
3160
|
Google::Apis::BigqueryV2::Job.new(
|
|
2967
3161
|
job_reference: job_ref,
|
|
@@ -2973,43 +3167,63 @@ module Google
|
|
|
2973
3167
|
table_id: table_id
|
|
2974
3168
|
)
|
|
2975
3169
|
),
|
|
2976
|
-
dry_run: dryrun
|
|
3170
|
+
dry_run: dryrun,
|
|
3171
|
+
reservation: reservation
|
|
2977
3172
|
)
|
|
2978
3173
|
)
|
|
2979
3174
|
end
|
|
2980
3175
|
|
|
2981
3176
|
def load_job_csv_options! job, jagged_rows: nil, quoted_newlines: nil, delimiter: nil, quote: nil,
|
|
2982
|
-
skip_leading: nil, null_marker: nil
|
|
3177
|
+
skip_leading: nil, null_marker: nil, null_markers: nil, source_column_match: nil,
|
|
3178
|
+
preserve_ascii_control_characters: nil
|
|
2983
3179
|
job.jagged_rows = jagged_rows unless jagged_rows.nil?
|
|
2984
3180
|
job.quoted_newlines = quoted_newlines unless quoted_newlines.nil?
|
|
2985
3181
|
job.delimiter = delimiter unless delimiter.nil?
|
|
2986
3182
|
job.null_marker = null_marker unless null_marker.nil?
|
|
2987
3183
|
job.quote = quote unless quote.nil?
|
|
2988
3184
|
job.skip_leading = skip_leading unless skip_leading.nil?
|
|
3185
|
+
job.null_markers = null_markers unless null_markers.nil?
|
|
3186
|
+
job.source_column_match = source_column_match unless source_column_match.nil?
|
|
3187
|
+
job.preserve_ascii_control_characters = preserve_ascii_control_characters unless
|
|
3188
|
+
preserve_ascii_control_characters.nil?
|
|
2989
3189
|
end
|
|
2990
3190
|
|
|
2991
3191
|
def load_job_file_options! job, format: nil, projection_fields: nil, jagged_rows: nil, quoted_newlines: nil,
|
|
2992
3192
|
encoding: nil, delimiter: nil, ignore_unknown: nil, max_bad_records: nil, quote: nil,
|
|
2993
|
-
skip_leading: nil, null_marker: nil
|
|
3193
|
+
skip_leading: nil, null_marker: nil, date_format: nil, datetime_format: nil,
|
|
3194
|
+
time_format: nil, timestamp_format: nil, null_markers: nil, source_column_match: nil,
|
|
3195
|
+
time_zone: nil, reference_file_schema_uri: nil,
|
|
3196
|
+
preserve_ascii_control_characters: nil
|
|
2994
3197
|
job.format = format unless format.nil?
|
|
2995
3198
|
job.projection_fields = projection_fields unless projection_fields.nil?
|
|
2996
3199
|
job.encoding = encoding unless encoding.nil?
|
|
2997
3200
|
job.ignore_unknown = ignore_unknown unless ignore_unknown.nil?
|
|
2998
3201
|
job.max_bad_records = max_bad_records unless max_bad_records.nil?
|
|
3202
|
+
job.date_format = date_format unless date_format.nil?
|
|
3203
|
+
job.datetime_format = datetime_format unless datetime_format.nil?
|
|
3204
|
+
job.time_format = time_format unless time_format.nil?
|
|
3205
|
+
job.timestamp_format = timestamp_format unless timestamp_format.nil?
|
|
3206
|
+
job.time_zone = time_zone unless time_zone.nil?
|
|
3207
|
+
job.reference_file_schema_uri = reference_file_schema_uri unless reference_file_schema_uri.nil?
|
|
2999
3208
|
load_job_csv_options! job, jagged_rows: jagged_rows,
|
|
3000
3209
|
quoted_newlines: quoted_newlines,
|
|
3001
3210
|
delimiter: delimiter,
|
|
3002
3211
|
quote: quote,
|
|
3003
3212
|
skip_leading: skip_leading,
|
|
3004
|
-
null_marker: null_marker
|
|
3213
|
+
null_marker: null_marker,
|
|
3214
|
+
null_markers: null_markers,
|
|
3215
|
+
source_column_match: source_column_match,
|
|
3216
|
+
preserve_ascii_control_characters: preserve_ascii_control_characters
|
|
3005
3217
|
end
|
|
3006
3218
|
|
|
3007
3219
|
def load_job_updater table_id, format: nil, create: nil, write: nil, projection_fields: nil, jagged_rows: nil,
|
|
3008
3220
|
quoted_newlines: nil, encoding: nil, delimiter: nil, ignore_unknown: nil,
|
|
3009
3221
|
max_bad_records: nil, quote: nil, skip_leading: nil, dryrun: nil, schema: nil, job_id: nil,
|
|
3010
3222
|
prefix: nil, labels: nil, autodetect: nil, null_marker: nil, create_session: nil,
|
|
3011
|
-
session_id: nil
|
|
3012
|
-
|
|
3223
|
+
session_id: nil, date_format: nil, datetime_format: nil, time_format: nil,
|
|
3224
|
+
timestamp_format: nil, null_markers: nil, source_column_match: nil, time_zone: nil,
|
|
3225
|
+
reference_file_schema_uri: nil, preserve_ascii_control_characters: nil, reservation: nil
|
|
3226
|
+
new_job = load_job_gapi table_id, dryrun, job_id: job_id, prefix: prefix, reservation: reservation
|
|
3013
3227
|
LoadJob::Updater.new(new_job).tap do |job|
|
|
3014
3228
|
job.location = location if location # may be dataset reference
|
|
3015
3229
|
job.create = create unless create.nil?
|
|
@@ -3029,7 +3243,16 @@ module Google
|
|
|
3029
3243
|
max_bad_records: max_bad_records,
|
|
3030
3244
|
quote: quote,
|
|
3031
3245
|
skip_leading: skip_leading,
|
|
3032
|
-
null_marker: null_marker
|
|
3246
|
+
null_marker: null_marker,
|
|
3247
|
+
date_format: date_format,
|
|
3248
|
+
datetime_format: datetime_format,
|
|
3249
|
+
time_format: time_format,
|
|
3250
|
+
timestamp_format: timestamp_format,
|
|
3251
|
+
null_markers: null_markers,
|
|
3252
|
+
source_column_match: source_column_match,
|
|
3253
|
+
time_zone: time_zone,
|
|
3254
|
+
reference_file_schema_uri: reference_file_schema_uri,
|
|
3255
|
+
preserve_ascii_control_characters: preserve_ascii_control_characters
|
|
3033
3256
|
end
|
|
3034
3257
|
end
|
|
3035
3258
|
|
|
@@ -280,7 +280,8 @@ module Google
|
|
|
280
280
|
job_reference: job_ref,
|
|
281
281
|
configuration: Google::Apis::BigqueryV2::JobConfiguration.new(
|
|
282
282
|
extract: extract_config,
|
|
283
|
-
dry_run: options[:dryrun]
|
|
283
|
+
dry_run: options[:dryrun],
|
|
284
|
+
reservation: options[:reservation]
|
|
284
285
|
)
|
|
285
286
|
)
|
|
286
287
|
|
|
@@ -438,6 +439,18 @@ module Google
|
|
|
438
439
|
@gapi.configuration.extract.use_avro_logical_types = value
|
|
439
440
|
end
|
|
440
441
|
|
|
442
|
+
##
|
|
443
|
+
# Sets the reservation that job would use. User can specify a reservation
|
|
444
|
+
# to execute the job. If reservation is not set, reservation is determined
|
|
445
|
+
# based on the rules defined by the reservation assignments. The expected
|
|
446
|
+
# format is `projects/`project`/locations/`location`/reservations/`reservation``.
|
|
447
|
+
# @param [String] value The reservation name.
|
|
448
|
+
#
|
|
449
|
+
# @!group Attributes
|
|
450
|
+
def reservation= value
|
|
451
|
+
@gapi.configuration.update! reservation: value
|
|
452
|
+
end
|
|
453
|
+
|
|
441
454
|
def cancel
|
|
442
455
|
raise "not implemented in #{self.class}"
|
|
443
456
|
end
|
|
@@ -737,7 +737,7 @@ module Google
|
|
|
737
737
|
end
|
|
738
738
|
|
|
739
739
|
##
|
|
740
|
-
# The URI of
|
|
740
|
+
# The URI of the reference file with the reader schema. This file is only
|
|
741
741
|
# loaded if it is part of source URIs, but is not loaded otherwise.
|
|
742
742
|
# It is enabled for the following formats: `AVRO`, `PARQUET`, `ORC`.
|
|
743
743
|
#
|
|
@@ -2779,7 +2779,7 @@ module Google
|
|
|
2779
2779
|
|
|
2780
2780
|
##
|
|
2781
2781
|
|
|
2782
|
-
# Sets the URI of
|
|
2782
|
+
# Sets the URI of the reference file with the reader schema. This file
|
|
2783
2783
|
# is only loaded if it is part of source URIs, but is not loaded
|
|
2784
2784
|
# otherwise. It is enabled for the following formats: `AVRO`,
|
|
2785
2785
|
# `PARQUET`, `ORC`.
|
|
@@ -2800,6 +2800,18 @@ module Google
|
|
|
2800
2800
|
@gapi.configuration.load.update! preserve_ascii_control_characters: val
|
|
2801
2801
|
end
|
|
2802
2802
|
|
|
2803
|
+
##
|
|
2804
|
+
# Sets the reservation that job would use. User can specify a reservation
|
|
2805
|
+
# to execute the job. If reservation is not set, reservation is determined
|
|
2806
|
+
# based on the rules defined by the reservation assignments. The expected
|
|
2807
|
+
# format is `projects/`project`/locations/`location`/reservations/`reservation``.
|
|
2808
|
+
# @param [String] value The reservation name.
|
|
2809
|
+
#
|
|
2810
|
+
# @!group Attributes
|
|
2811
|
+
def reservation= value
|
|
2812
|
+
@gapi.configuration.update! reservation: value
|
|
2813
|
+
end
|
|
2814
|
+
|
|
2803
2815
|
def cancel
|
|
2804
2816
|
raise "not implemented in #{self.class}"
|
|
2805
2817
|
end
|
|
@@ -545,6 +545,11 @@ module Google
|
|
|
545
545
|
# * The key portion of a label must be unique. However, you can use the
|
|
546
546
|
# same key with multiple resources.
|
|
547
547
|
# * Keys must start with a lowercase letter or international character.
|
|
548
|
+
# @param [String] reservation The reservation that job would use. User
|
|
549
|
+
# can specify a reservation to execute the job. If reservation is not
|
|
550
|
+
# set, reservation is determined based on the rules defined by the
|
|
551
|
+
# reservation assignments. The expected format is
|
|
552
|
+
# `projects/`project`/locations/`location`/reservations/`reservation``.
|
|
548
553
|
#
|
|
549
554
|
# @yield [job] a job configuration object
|
|
550
555
|
# @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
|
|
@@ -566,9 +571,9 @@ module Google
|
|
|
566
571
|
#
|
|
567
572
|
# @!group Data
|
|
568
573
|
#
|
|
569
|
-
def extract_job extract_url, format: nil, job_id: nil, prefix: nil, labels: nil
|
|
574
|
+
def extract_job extract_url, format: nil, job_id: nil, prefix: nil, labels: nil, reservation: nil
|
|
570
575
|
ensure_service!
|
|
571
|
-
options = { format: format, job_id: job_id, prefix: prefix, labels: labels }
|
|
576
|
+
options = { format: format, job_id: job_id, prefix: prefix, labels: labels, reservation: reservation }
|
|
572
577
|
updater = ExtractJob::Updater.from_options service, model_ref, extract_url, options
|
|
573
578
|
updater.location = location if location # may be model reference
|
|
574
579
|
|
|
@@ -603,6 +608,12 @@ module Google
|
|
|
603
608
|
#
|
|
604
609
|
# * `ml_tf_saved_model` - TensorFlow SavedModel
|
|
605
610
|
# * `ml_xgboost_booster` - XGBoost Booster
|
|
611
|
+
# @param [String] reservation The reservation that job would use. User
|
|
612
|
+
# can specify a reservation to execute the job. If reservation is not
|
|
613
|
+
# set, reservation is determined based on the rules defined by the
|
|
614
|
+
# reservation assignments. The expected format is
|
|
615
|
+
# `projects/`project`/locations/`location`/reservations/`reservation``.
|
|
616
|
+
#
|
|
606
617
|
# @yield [job] a job configuration object
|
|
607
618
|
# @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
|
|
608
619
|
# configuration object for setting additional options.
|
|
@@ -620,8 +631,8 @@ module Google
|
|
|
620
631
|
#
|
|
621
632
|
# @!group Data
|
|
622
633
|
#
|
|
623
|
-
def extract extract_url, format: nil, &block
|
|
624
|
-
job = extract_job extract_url, format: format, &block
|
|
634
|
+
def extract extract_url, format: nil, reservation: nil, &block
|
|
635
|
+
job = extract_job extract_url, format: format, reservation: reservation, &block
|
|
625
636
|
job.wait_until_done!
|
|
626
637
|
ensure_job_succeeded! job
|
|
627
638
|
true
|