google-cloud-bigquery 1.21.2 → 1.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +54 -0
- data/CONTRIBUTING.md +1 -1
- data/lib/google-cloud-bigquery.rb +1 -0
- data/lib/google/cloud/bigquery.rb +1 -1
- data/lib/google/cloud/bigquery/convert.rb +3 -1
- data/lib/google/cloud/bigquery/copy_job.rb +15 -6
- data/lib/google/cloud/bigquery/dataset.rb +43 -20
- data/lib/google/cloud/bigquery/dataset/access.rb +293 -16
- data/lib/google/cloud/bigquery/external.rb +328 -3
- data/lib/google/cloud/bigquery/extract_job.rb +154 -50
- data/lib/google/cloud/bigquery/load_job.rb +197 -34
- data/lib/google/cloud/bigquery/model.rb +164 -8
- data/lib/google/cloud/bigquery/policy.rb +431 -0
- data/lib/google/cloud/bigquery/project.rb +137 -68
- data/lib/google/cloud/bigquery/query_job.rb +24 -12
- data/lib/google/cloud/bigquery/service.rb +50 -11
- data/lib/google/cloud/bigquery/table.rb +174 -37
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +7 -6
@@ -0,0 +1,431 @@
|
|
1
|
+
# Copyright 2020 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
require "google/apis/bigquery_v2"
|
17
|
+
|
18
|
+
module Google
|
19
|
+
module Cloud
|
20
|
+
module Bigquery
|
21
|
+
##
|
22
|
+
# # Policy
|
23
|
+
#
|
24
|
+
# Represents a Cloud IAM Policy for BigQuery resources.
|
25
|
+
#
|
26
|
+
# A Policy is a collection of bindings. A {Policy::Binding} binds one or more members to a single role. Member
|
27
|
+
# strings can describe user accounts, service accounts, Google groups, and domains. A role string represents a
|
28
|
+
# named list of permissions; each role can be an IAM predefined role or a user-created custom role.
|
29
|
+
#
|
30
|
+
# @see https://cloud.google.com/iam/docs/managing-policies Managing Policies
|
31
|
+
# @see https://cloud.google.com/bigquery/docs/table-access-controls-intro Controlling access to tables
|
32
|
+
#
|
33
|
+
# @attr [String] etag Used to check if the policy has changed since the last request. When you make a request with
|
34
|
+
# an `etag` value, Cloud IAM compares the `etag` value in the request with the existing `etag` value associated
|
35
|
+
# with the policy. It writes the policy only if the `etag` values match.
|
36
|
+
# @attr [Array<Binding>] bindings The bindings in the policy, which may be mutable or frozen depending on the
|
37
|
+
# context. See [Understanding Roles](https://cloud.google.com/iam/docs/understanding-roles) for a list of
|
38
|
+
# primitive and curated roles. See [BigQuery Table ACL
|
39
|
+
# permissions](https://cloud.google.com/bigquery/docs/table-access-controls-intro#permissions) for a list of
|
40
|
+
# values and patterns for members.
|
41
|
+
#
|
42
|
+
# @example
|
43
|
+
# require "google/cloud/bigquery"
|
44
|
+
#
|
45
|
+
# bigquery = Google::Cloud::Bigquery.new
|
46
|
+
# dataset = bigquery.dataset "my_dataset"
|
47
|
+
# table = dataset.table "my_table"
|
48
|
+
# policy = table.policy
|
49
|
+
#
|
50
|
+
# policy.frozen? #=> true
|
51
|
+
# binding_owner = policy.bindings.find { |b| b.role == "roles/owner" }
|
52
|
+
#
|
53
|
+
# binding_owner.role #=> "roles/owner"
|
54
|
+
# binding_owner.members #=> ["user:owner@example.com"]
|
55
|
+
# binding_owner.frozen? #=> true
|
56
|
+
# binding_owner.members.frozen? #=> true
|
57
|
+
#
|
58
|
+
# @example Update mutable bindings in the policy.
|
59
|
+
# require "google/cloud/bigquery"
|
60
|
+
#
|
61
|
+
# bigquery = Google::Cloud::Bigquery.new
|
62
|
+
# dataset = bigquery.dataset "my_dataset"
|
63
|
+
# table = dataset.table "my_table"
|
64
|
+
#
|
65
|
+
# table.update_policy do |p|
|
66
|
+
# p.grant role: "roles/viewer", members: "user:viewer@example.com"
|
67
|
+
# p.revoke role: "roles/editor", members: "user:editor@example.com"
|
68
|
+
# p.revoke role: "roles/owner"
|
69
|
+
# end
|
70
|
+
#
|
71
|
+
# @example Iterate over frozen bindings.
|
72
|
+
# require "google/cloud/bigquery"
|
73
|
+
#
|
74
|
+
# bigquery = Google::Cloud::Bigquery.new
|
75
|
+
# dataset = bigquery.dataset "my_dataset"
|
76
|
+
# table = dataset.table "my_table"
|
77
|
+
# policy = table.policy
|
78
|
+
#
|
79
|
+
# policy.frozen? #=> true
|
80
|
+
# policy.bindings.each do |b|
|
81
|
+
# puts b.role
|
82
|
+
# puts b.members
|
83
|
+
# end
|
84
|
+
#
|
85
|
+
# @example Update mutable bindings.
|
86
|
+
# require "google/cloud/bigquery"
|
87
|
+
#
|
88
|
+
# bigquery = Google::Cloud::Bigquery.new
|
89
|
+
# dataset = bigquery.dataset "my_dataset"
|
90
|
+
# table = dataset.table "my_table"
|
91
|
+
#
|
92
|
+
# table.update_policy do |p|
|
93
|
+
# p.bindings.each do |b|
|
94
|
+
# b.members.delete_if { |m| m.include? "@example.com" }
|
95
|
+
# end
|
96
|
+
# end
|
97
|
+
#
|
98
|
+
class Policy
|
99
|
+
attr_reader :etag, :bindings
|
100
|
+
|
101
|
+
# @private
|
102
|
+
def initialize etag, bindings
|
103
|
+
@etag = etag.freeze
|
104
|
+
@bindings = bindings
|
105
|
+
end
|
106
|
+
|
107
|
+
##
|
108
|
+
# Convenience method adding or updating a binding in the policy. See [Understanding
|
109
|
+
# Roles](https://cloud.google.com/iam/docs/understanding-roles) for a list of primitive and curated roles. See
|
110
|
+
# [BigQuery Table ACL
|
111
|
+
# permissions](https://cloud.google.com/bigquery/docs/table-access-controls-intro#permissions) for a list of
|
112
|
+
# values and patterns for members.
|
113
|
+
#
|
114
|
+
# @param [String] role The role that is bound to members in the binding. For example, `roles/viewer`,
|
115
|
+
# `roles/editor`, or `roles/owner`. Required.
|
116
|
+
# @param [String, Array<String>] members Specifies the identities requesting access for a Cloud Platform
|
117
|
+
# resource. `members` can have the following values. Required.
|
118
|
+
#
|
119
|
+
# * `allUsers`: A special identifier that represents anyone who is on the internet; with or without a Google
|
120
|
+
# account.
|
121
|
+
# * `allAuthenticatedUsers`: A special identifier that represents anyone who is authenticated with a Google
|
122
|
+
# account or a service account.
|
123
|
+
# * `user:<emailid>`: An email address that represents a specific Google account. For example,
|
124
|
+
# `alice@example.com`.
|
125
|
+
# * `serviceAccount:<emailid>`: An email address that represents a service account. For example,
|
126
|
+
# `my-other-app@appspot.gserviceaccount.com`.
|
127
|
+
# * `group:<emailid>`: An email address that represents a Google group. For example, `admins@example.com`.
|
128
|
+
# * `deleted:user:<emailid>?uid=<uniqueid>`: An email address (plus unique identifier) representing a user
|
129
|
+
# that has been recently deleted. For example, `alice@example.com?uid=123456789012345678901`. If the user
|
130
|
+
# is recovered, this value reverts to `user:<emailid>` and the recovered user retains the role in the
|
131
|
+
# binding.
|
132
|
+
# * `deleted: serviceAccount:<emailid>?uid=<uniqueid>`: An email address (plus unique identifier) representing
|
133
|
+
# a service account that has been recently deleted. For example,
|
134
|
+
# `my-other-app@appspot.gserviceaccount.com?uid=123456789012345678901`. If the service account is undeleted,
|
135
|
+
# this value reverts to `serviceAccount:<emailid>` and the undeleted service account retains the role in
|
136
|
+
# the binding.
|
137
|
+
# * `deleted:group:<emailid>?uid=<uniqueid>`: An email address (plus unique identifier) representing a Google
|
138
|
+
# group that has been recently deleted. For example, `admins@example.com?uid=123456789012345678901`. If the
|
139
|
+
# group is recovered, this value reverts to `group:<emailid>` and the recovered group retains the role in
|
140
|
+
# the binding.
|
141
|
+
# * `domain:<domain>`: The G Suite domain (primary) that represents all the users of that domain. For example,
|
142
|
+
# `google.com` or `example.com`.
|
143
|
+
#
|
144
|
+
# @return [nil]
|
145
|
+
#
|
146
|
+
# @example Grant a role to a member.
|
147
|
+
# require "google/cloud/bigquery"
|
148
|
+
#
|
149
|
+
# bigquery = Google::Cloud::Bigquery.new
|
150
|
+
# dataset = bigquery.dataset "my_dataset"
|
151
|
+
# table = dataset.table "my_table"
|
152
|
+
#
|
153
|
+
# table.update_policy do |p|
|
154
|
+
# p.grant role: "roles/viewer", members: "user:viewer@example.com"
|
155
|
+
# end
|
156
|
+
#
|
157
|
+
def grant role:, members:
|
158
|
+
existing_binding = bindings.find { |b| b.role == role }
|
159
|
+
if existing_binding
|
160
|
+
existing_binding.members.concat Array(members)
|
161
|
+
existing_binding.members.uniq!
|
162
|
+
else
|
163
|
+
bindings << Binding.new(role, members)
|
164
|
+
end
|
165
|
+
nil
|
166
|
+
end
|
167
|
+
|
168
|
+
##
|
169
|
+
# Convenience method for removing a binding or bindings from the policy. See
|
170
|
+
# [Understanding Roles](https://cloud.google.com/iam/docs/understanding-roles) for a list of primitive and
|
171
|
+
# curated roles. See [BigQuery Table ACL
|
172
|
+
# permissions](https://cloud.google.com/bigquery/docs/table-access-controls-intro#permissions) for a list of
|
173
|
+
# values and patterns for members.
|
174
|
+
#
|
175
|
+
# @param [String] role A role that is bound to members in the policy. For example, `roles/viewer`,
|
176
|
+
# `roles/editor`, or `roles/owner`. Optional.
|
177
|
+
# @param [String, Array<String>] members Specifies the identities receiving access for a Cloud Platform
|
178
|
+
# resource. `members` can have the following values. Optional.
|
179
|
+
#
|
180
|
+
# * `allUsers`: A special identifier that represents anyone who is on the internet; with or without a Google
|
181
|
+
# account.
|
182
|
+
# * `allAuthenticatedUsers`: A special identifier that represents anyone who is authenticated with a Google
|
183
|
+
# account or a service account.
|
184
|
+
# * `user:<emailid>`: An email address that represents a specific Google account. For example,
|
185
|
+
# `alice@example.com`.
|
186
|
+
# * `serviceAccount:<emailid>`: An email address that represents a service account. For example,
|
187
|
+
# `my-other-app@appspot.gserviceaccount.com`.
|
188
|
+
# * `group:<emailid>`: An email address that represents a Google group. For example, `admins@example.com`.
|
189
|
+
# * `deleted:user:<emailid>?uid=<uniqueid>`: An email address (plus unique identifier) representing a user
|
190
|
+
# that has been recently deleted. For example, `alice@example.com?uid=123456789012345678901`. If the user
|
191
|
+
# is recovered, this value reverts to `user:<emailid>` and the recovered user retains the role in the
|
192
|
+
# binding.
|
193
|
+
# * `deleted: serviceAccount:<emailid>?uid=<uniqueid>`: An email address (plus unique identifier) representing
|
194
|
+
# a service account that has been recently deleted. For example,
|
195
|
+
# `my-other-app@appspot.gserviceaccount.com?uid=123456789012345678901`. If the service account is undeleted,
|
196
|
+
# this value reverts to `serviceAccount:<emailid>` and the undeleted service account retains the role in
|
197
|
+
# the binding.
|
198
|
+
# * `deleted:group:<emailid>?uid=<uniqueid>`: An email address (plus unique identifier) representing a Google
|
199
|
+
# group that has been recently deleted. For example, `admins@example.com?uid=123456789012345678901`. If the
|
200
|
+
# group is recovered, this value reverts to `group:<emailid>` and the recovered group retains the role in
|
201
|
+
# the binding.
|
202
|
+
# * `domain:<domain>`: The G Suite domain (primary) that represents all the users of that domain. For example,
|
203
|
+
# `google.com` or `example.com`.
|
204
|
+
#
|
205
|
+
# @return [nil]
|
206
|
+
#
|
207
|
+
# @example Revoke a role for a member or members.
|
208
|
+
# require "google/cloud/bigquery"
|
209
|
+
#
|
210
|
+
# bigquery = Google::Cloud::Bigquery.new
|
211
|
+
# dataset = bigquery.dataset "my_dataset"
|
212
|
+
# table = dataset.table "my_table"
|
213
|
+
#
|
214
|
+
# table.update_policy do |p|
|
215
|
+
# p.revoke role: "roles/viewer", members: "user:viewer@example.com"
|
216
|
+
# end
|
217
|
+
#
|
218
|
+
# @example Revoke a role for all members.
|
219
|
+
# require "google/cloud/bigquery"
|
220
|
+
#
|
221
|
+
# bigquery = Google::Cloud::Bigquery.new
|
222
|
+
# dataset = bigquery.dataset "my_dataset"
|
223
|
+
# table = dataset.table "my_table"
|
224
|
+
#
|
225
|
+
# table.update_policy do |p|
|
226
|
+
# p.revoke role: "roles/viewer"
|
227
|
+
# end
|
228
|
+
#
|
229
|
+
# @example Revoke all roles for a member or members.
|
230
|
+
# require "google/cloud/bigquery"
|
231
|
+
#
|
232
|
+
# bigquery = Google::Cloud::Bigquery.new
|
233
|
+
# dataset = bigquery.dataset "my_dataset"
|
234
|
+
# table = dataset.table "my_table"
|
235
|
+
#
|
236
|
+
# table.update_policy do |p|
|
237
|
+
# p.revoke members: ["user:viewer@example.com", "user:editor@example.com"]
|
238
|
+
# end
|
239
|
+
#
|
240
|
+
def revoke role: nil, members: nil
|
241
|
+
bindings_for_role = role ? bindings.select { |b| b.role == role } : bindings
|
242
|
+
bindings_for_role.each do |b|
|
243
|
+
if members
|
244
|
+
b.members -= Array(members)
|
245
|
+
bindings.delete b if b.members.empty?
|
246
|
+
else
|
247
|
+
bindings.delete b
|
248
|
+
end
|
249
|
+
end
|
250
|
+
nil
|
251
|
+
end
|
252
|
+
|
253
|
+
##
|
254
|
+
# @private Convert the Policy to a Google::Apis::BigqueryV2::Policy.
|
255
|
+
def to_gapi
|
256
|
+
Google::Apis::BigqueryV2::Policy.new(
|
257
|
+
bindings: bindings_to_gapi,
|
258
|
+
etag: etag,
|
259
|
+
version: 1
|
260
|
+
)
|
261
|
+
end
|
262
|
+
|
263
|
+
##
|
264
|
+
# @private Deep freeze the policy including its bindings.
|
265
|
+
def freeze
|
266
|
+
super
|
267
|
+
@bindings.each(&:freeze)
|
268
|
+
@bindings.freeze
|
269
|
+
self
|
270
|
+
end
|
271
|
+
|
272
|
+
##
|
273
|
+
# @private New Policy from a Google::Apis::BigqueryV2::Policy object.
|
274
|
+
def self.from_gapi gapi
|
275
|
+
bindings = Array(gapi.bindings).map do |binding|
|
276
|
+
Binding.new binding.role, binding.members.to_a
|
277
|
+
end
|
278
|
+
new gapi.etag, bindings
|
279
|
+
end
|
280
|
+
|
281
|
+
##
|
282
|
+
# # Policy::Binding
|
283
|
+
#
|
284
|
+
# Represents a Cloud IAM Binding for BigQuery resources within the context of a {Policy}.
|
285
|
+
#
|
286
|
+
# A binding binds one or more members to a single role. Member strings can describe user accounts, service
|
287
|
+
# accounts, Google groups, and domains. A role is a named list of permissions; each role can be an IAM
|
288
|
+
# predefined role or a user-created custom role.
|
289
|
+
#
|
290
|
+
# @see https://cloud.google.com/bigquery/docs/table-access-controls-intro Controlling access to tables
|
291
|
+
#
|
292
|
+
# @attr [String] role The role that is assigned to `members`. For example, `roles/viewer`, `roles/editor`, or
|
293
|
+
# `roles/owner`. Required.
|
294
|
+
# @attr [Array<String>] members Specifies the identities requesting access for a Cloud Platform resource.
|
295
|
+
# `members` can have the following values. Required.
|
296
|
+
#
|
297
|
+
# * `allUsers`: A special identifier that represents anyone who is on the internet; with or without a Google
|
298
|
+
# account.
|
299
|
+
# * `allAuthenticatedUsers`: A special identifier that represents anyone who is authenticated with a Google
|
300
|
+
# account or a service account.
|
301
|
+
# * `user:<emailid>`: An email address that represents a specific Google account. For example,
|
302
|
+
# `alice@example.com`.
|
303
|
+
# * `serviceAccount:<emailid>`: An email address that represents a service account. For example,
|
304
|
+
# `my-other-app@appspot.gserviceaccount.com`.
|
305
|
+
# * `group:<emailid>`: An email address that represents a Google group. For example, `admins@example.com`.
|
306
|
+
# * `deleted:user:<emailid>?uid=<uniqueid>`: An email address (plus unique identifier) representing a user
|
307
|
+
# that has been recently deleted. For example, `alice@example.com?uid=123456789012345678901`. If the user
|
308
|
+
# is recovered, this value reverts to `user:<emailid>` and the recovered user retains the role in the
|
309
|
+
# binding.
|
310
|
+
# * `deleted: serviceAccount:<emailid>?uid=<uniqueid>`: An email address (plus unique identifier) representing
|
311
|
+
# a service account that has been recently deleted. For example,
|
312
|
+
# `my-other-app@appspot.gserviceaccount.com?uid=123456789012345678901`. If the service account is undeleted,
|
313
|
+
# this value reverts to `serviceAccount:<emailid>` and the undeleted service account retains the role in
|
314
|
+
# the binding.
|
315
|
+
# * `deleted:group:<emailid>?uid=<uniqueid>`: An email address (plus unique identifier) representing a Google
|
316
|
+
# group that has been recently deleted. For example, `admins@example.com?uid=123456789012345678901`. If the
|
317
|
+
# group is recovered, this value reverts to `group:<emailid>` and the recovered group retains the role in
|
318
|
+
# the binding.
|
319
|
+
# * `domain:<domain>`: The G Suite domain (primary) that represents all the users of that domain. For example,
|
320
|
+
# `google.com` or `example.com`.
|
321
|
+
#
|
322
|
+
# @example
|
323
|
+
# require "google/cloud/bigquery"
|
324
|
+
#
|
325
|
+
# bigquery = Google::Cloud::Bigquery.new
|
326
|
+
# dataset = bigquery.dataset "my_dataset"
|
327
|
+
# table = dataset.table "my_table"
|
328
|
+
#
|
329
|
+
# policy = table.policy
|
330
|
+
# binding_owner = policy.bindings.find { |b| b.role == "roles/owner" }
|
331
|
+
#
|
332
|
+
# binding_owner.role #=> "roles/owner"
|
333
|
+
# binding_owner.members #=> ["user:owner@example.com"]
|
334
|
+
#
|
335
|
+
# binding_owner.frozen? #=> true
|
336
|
+
# binding_owner.members.frozen? #=> true
|
337
|
+
#
|
338
|
+
# @example Update mutable bindings.
|
339
|
+
# require "google/cloud/bigquery"
|
340
|
+
#
|
341
|
+
# bigquery = Google::Cloud::Bigquery.new
|
342
|
+
# dataset = bigquery.dataset "my_dataset"
|
343
|
+
# table = dataset.table "my_table"
|
344
|
+
#
|
345
|
+
# table.update_policy do |p|
|
346
|
+
# binding_owner = p.bindings.find { |b| b.role == "roles/owner" }
|
347
|
+
# binding_owner.members.delete_if { |m| m.include? "@example.com" }
|
348
|
+
# end
|
349
|
+
#
|
350
|
+
class Binding
|
351
|
+
attr_accessor :role
|
352
|
+
attr_reader :members
|
353
|
+
|
354
|
+
# @private
|
355
|
+
def initialize role, members
|
356
|
+
members = Array(members).uniq
|
357
|
+
raise ArgumentError, "members cannot be empty" if members.empty?
|
358
|
+
@role = role
|
359
|
+
@members = members
|
360
|
+
end
|
361
|
+
|
362
|
+
##
|
363
|
+
# Sets the binding members.
|
364
|
+
#
|
365
|
+
# @param [Array<String>] new_members Specifies the identities requesting access for a Cloud Platform resource.
|
366
|
+
# `new_members` can have the following values. Required.
|
367
|
+
#
|
368
|
+
# * `allUsers`: A special identifier that represents anyone who is on the internet; with or without a Google
|
369
|
+
# account.
|
370
|
+
# * `allAuthenticatedUsers`: A special identifier that represents anyone who is authenticated with a Google
|
371
|
+
# account or a service account.
|
372
|
+
# * `user:<emailid>`: An email address that represents a specific Google account. For example,
|
373
|
+
# `alice@example.com`.
|
374
|
+
# * `serviceAccount:<emailid>`: An email address that represents a service account. For example,
|
375
|
+
# `my-other-app@appspot.gserviceaccount.com`.
|
376
|
+
# * `group:<emailid>`: An email address that represents a Google group. For example, `admins@example.com`.
|
377
|
+
# * `deleted:user:<emailid>?uid=<uniqueid>`: An email address (plus unique identifier) representing a user
|
378
|
+
# that has been recently deleted. For example, `alice@example.com?uid=123456789012345678901`. If the user
|
379
|
+
# is recovered, this value reverts to `user:<emailid>` and the recovered user retains the role in the
|
380
|
+
# binding.
|
381
|
+
# * `deleted: serviceAccount:<emailid>?uid=<uniqueid>`: An email address (plus unique identifier)
|
382
|
+
# representing a service account that has been recently deleted. For example,
|
383
|
+
# `my-other-app@appspot.gserviceaccount.com?uid=123456789012345678901`. If the service account is
|
384
|
+
# undeleted, this value reverts to `serviceAccount:<emailid>` and the undeleted service account retains
|
385
|
+
# the role in the binding.
|
386
|
+
# * `deleted:group:<emailid>?uid=<uniqueid>`: An email address (plus unique identifier) representing a
|
387
|
+
# Google group that has been recently deleted. For example,
|
388
|
+
# `admins@example.com?uid=123456789012345678901`. If the group is recovered, this value reverts to
|
389
|
+
# `group:<emailid>` and the recovered group retains the role in the binding.
|
390
|
+
# * `domain:<domain>`: The G Suite domain (primary) that represents all the users of that domain. For
|
391
|
+
# example, `google.com` or `example.com`.
|
392
|
+
#
|
393
|
+
def members= new_members
|
394
|
+
@members = Array(new_members).uniq
|
395
|
+
end
|
396
|
+
|
397
|
+
##
|
398
|
+
# @private Convert the Binding to a Google::Apis::BigqueryV2::Binding.
|
399
|
+
def to_gapi
|
400
|
+
Google::Apis::BigqueryV2::Binding.new role: role, members: members
|
401
|
+
end
|
402
|
+
|
403
|
+
##
|
404
|
+
# @private Deep freeze the policy including its members.
|
405
|
+
def freeze
|
406
|
+
super
|
407
|
+
role.freeze
|
408
|
+
members.each(&:freeze)
|
409
|
+
members.freeze
|
410
|
+
self
|
411
|
+
end
|
412
|
+
|
413
|
+
##
|
414
|
+
# @private New Binding from a Google::Apis::BigqueryV2::Binding object.
|
415
|
+
def self.from_gapi gapi
|
416
|
+
new gapi.etag, gapi.members.to_a
|
417
|
+
end
|
418
|
+
end
|
419
|
+
|
420
|
+
protected
|
421
|
+
|
422
|
+
def bindings_to_gapi
|
423
|
+
@bindings.compact.uniq.map do |b|
|
424
|
+
next if b.members.empty?
|
425
|
+
b.to_gapi
|
426
|
+
end
|
427
|
+
end
|
428
|
+
end
|
429
|
+
end
|
430
|
+
end
|
431
|
+
end
|
@@ -153,13 +153,21 @@ module Google
|
|
153
153
|
# is 1,024 characters. If `job_id` is provided, then `prefix` will not
|
154
154
|
# be used.
|
155
155
|
# @param [Hash] labels A hash of user-provided labels associated with
|
156
|
-
# the job. You can use these to organize and group your jobs.
|
157
|
-
#
|
158
|
-
#
|
159
|
-
#
|
160
|
-
#
|
161
|
-
#
|
162
|
-
#
|
156
|
+
# the job. You can use these to organize and group your jobs.
|
157
|
+
#
|
158
|
+
# The labels applied to a resource must meet the following requirements:
|
159
|
+
#
|
160
|
+
# * Each resource can have multiple labels, up to a maximum of 64.
|
161
|
+
# * Each label must be a key-value pair.
|
162
|
+
# * Keys have a minimum length of 1 character and a maximum length of
|
163
|
+
# 63 characters, and cannot be empty. Values can be empty, and have
|
164
|
+
# a maximum length of 63 characters.
|
165
|
+
# * Keys and values can contain only lowercase letters, numeric characters,
|
166
|
+
# underscores, and dashes. All characters must use UTF-8 encoding, and
|
167
|
+
# international characters are allowed.
|
168
|
+
# * The key portion of a label must be unique. However, you can use the
|
169
|
+
# same key with multiple resources.
|
170
|
+
# * Keys must start with a lowercase letter or international character.
|
163
171
|
# @yield [job] a job configuration object
|
164
172
|
# @yieldparam [Google::Cloud::Bigquery::CopyJob::Updater] job a job
|
165
173
|
# configuration object for setting additional options.
|
@@ -411,13 +419,21 @@ module Google
|
|
411
419
|
# See [Generating a job
|
412
420
|
# ID](https://cloud.google.com/bigquery/docs/managing-jobs#generate-jobid).
|
413
421
|
# @param [Hash] labels A hash of user-provided labels associated with
|
414
|
-
# the job. You can use these to organize and group your jobs.
|
415
|
-
#
|
416
|
-
#
|
417
|
-
#
|
418
|
-
#
|
419
|
-
#
|
420
|
-
#
|
422
|
+
# the job. You can use these to organize and group your jobs.
|
423
|
+
#
|
424
|
+
# The labels applied to a resource must meet the following requirements:
|
425
|
+
#
|
426
|
+
# * Each resource can have multiple labels, up to a maximum of 64.
|
427
|
+
# * Each label must be a key-value pair.
|
428
|
+
# * Keys have a minimum length of 1 character and a maximum length of
|
429
|
+
# 63 characters, and cannot be empty. Values can be empty, and have
|
430
|
+
# a maximum length of 63 characters.
|
431
|
+
# * Keys and values can contain only lowercase letters, numeric characters,
|
432
|
+
# underscores, and dashes. All characters must use UTF-8 encoding, and
|
433
|
+
# international characters are allowed.
|
434
|
+
# * The key portion of a label must be unique. However, you can use the
|
435
|
+
# same key with multiple resources.
|
436
|
+
# * Keys must start with a lowercase letter or international character.
|
421
437
|
# @param [Array<String>, String] udfs User-defined function resources
|
422
438
|
# used in a legacy SQL query. May be either a code resource to load from
|
423
439
|
# a Google Cloud Storage URI (`gs://bucket/path`), or an inline resource
|
@@ -1445,46 +1461,58 @@ module Google
|
|
1445
1461
|
end
|
1446
1462
|
|
1447
1463
|
##
|
1448
|
-
# Extracts the data from
|
1449
|
-
#
|
1450
|
-
#
|
1451
|
-
# calling {Job#reload!} and {Job#done?} to detect when the job
|
1452
|
-
# or simply block until the job is done by calling
|
1464
|
+
# Extracts the data from a table or exports a model to Google Cloud Storage
|
1465
|
+
# asynchronously, immediately returning an {ExtractJob} that can be used to
|
1466
|
+
# track the progress of the export job. The caller may poll the service by
|
1467
|
+
# repeatedly calling {Job#reload!} and {Job#done?} to detect when the job
|
1468
|
+
# is done, or simply block until the job is done by calling
|
1453
1469
|
# #{Job#wait_until_done!}. See {#extract} for the synchronous version.
|
1454
|
-
#
|
1455
|
-
#
|
1470
|
+
#
|
1471
|
+
# Use this method instead of {Table#extract_job} or {Model#extract_job} to
|
1472
|
+
# extract data from source tables or models in other projects.
|
1456
1473
|
#
|
1457
1474
|
# The geographic location for the job ("US", "EU", etc.) can be set via
|
1458
1475
|
# {ExtractJob::Updater#location=} in a block passed to this method.
|
1459
1476
|
#
|
1460
|
-
# @see https://cloud.google.com/bigquery/exporting-data
|
1461
|
-
# Exporting
|
1477
|
+
# @see https://cloud.google.com/bigquery/docs/exporting-data
|
1478
|
+
# Exporting table data
|
1479
|
+
# @see https://cloud.google.com/bigquery-ml/docs/exporting-models
|
1480
|
+
# Exporting models
|
1462
1481
|
#
|
1463
|
-
# @param [
|
1464
|
-
#
|
1465
|
-
# [Standard SQL Query
|
1482
|
+
# @param [Table, Model, String] source The source table or model for
|
1483
|
+
# the extract operation. This can be a table or model object; or a
|
1484
|
+
# table ID string as specified by the [Standard SQL Query
|
1466
1485
|
# Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#from-clause)
|
1467
1486
|
# (`project-name.dataset_id.table_id`) or the [Legacy SQL Query
|
1468
1487
|
# Reference](https://cloud.google.com/bigquery/query-reference#from)
|
1469
1488
|
# (`project-name:dataset_id.table_id`).
|
1470
1489
|
# @param [Google::Cloud::Storage::File, String, Array<String>]
|
1471
1490
|
# extract_url The Google Storage file or file URI pattern(s) to which
|
1472
|
-
# BigQuery should extract
|
1473
|
-
#
|
1474
|
-
#
|
1491
|
+
# BigQuery should extract. For a model export this value should be a
|
1492
|
+
# string ending in an object name prefix, since multiple objects will
|
1493
|
+
# be exported.
|
1494
|
+
# @param [String] format The exported file format. The default value for
|
1495
|
+
# tables is `csv`. Tables with nested or repeated fields cannot be
|
1496
|
+
# exported as CSV. The default value for models is `ml_tf_saved_model`.
|
1475
1497
|
#
|
1476
|
-
#
|
1498
|
+
# Supported values for tables:
|
1477
1499
|
#
|
1478
1500
|
# * `csv` - CSV
|
1479
1501
|
# * `json` - [Newline-delimited JSON](http://jsonlines.org/)
|
1480
1502
|
# * `avro` - [Avro](http://avro.apache.org/)
|
1503
|
+
#
|
1504
|
+
# Supported values for models:
|
1505
|
+
#
|
1506
|
+
# * `ml_tf_saved_model` - TensorFlow SavedModel
|
1507
|
+
# * `ml_xgboost_booster` - XGBoost Booster
|
1481
1508
|
# @param [String] compression The compression type to use for exported
|
1482
1509
|
# files. Possible values include `GZIP` and `NONE`. The default value
|
1483
|
-
# is `NONE`.
|
1510
|
+
# is `NONE`. Not applicable when extracting models.
|
1484
1511
|
# @param [String] delimiter Delimiter to use between fields in the
|
1485
|
-
# exported data. Default is
|
1486
|
-
#
|
1487
|
-
#
|
1512
|
+
# exported table data. Default is `,`. Not applicable when extracting
|
1513
|
+
# models.
|
1514
|
+
# @param [Boolean] header Whether to print out a header row in table
|
1515
|
+
# exports. Default is `true`. Not applicable when extracting models.
|
1488
1516
|
# @param [String] job_id A user-defined ID for the extract job. The ID
|
1489
1517
|
# must contain only letters (a-z, A-Z), numbers (0-9), underscores
|
1490
1518
|
# (_), or dashes (-). The maximum length is 1,024 characters. If
|
@@ -1501,40 +1529,60 @@ module Google
|
|
1501
1529
|
# is 1,024 characters. If `job_id` is provided, then `prefix` will not
|
1502
1530
|
# be used.
|
1503
1531
|
# @param [Hash] labels A hash of user-provided labels associated with
|
1504
|
-
# the job. You can use these to organize and group your jobs.
|
1505
|
-
#
|
1506
|
-
#
|
1507
|
-
#
|
1508
|
-
#
|
1509
|
-
#
|
1510
|
-
#
|
1532
|
+
# the job. You can use these to organize and group your jobs.
|
1533
|
+
#
|
1534
|
+
# The labels applied to a resource must meet the following requirements:
|
1535
|
+
#
|
1536
|
+
# * Each resource can have multiple labels, up to a maximum of 64.
|
1537
|
+
# * Each label must be a key-value pair.
|
1538
|
+
# * Keys have a minimum length of 1 character and a maximum length of
|
1539
|
+
# 63 characters, and cannot be empty. Values can be empty, and have
|
1540
|
+
# a maximum length of 63 characters.
|
1541
|
+
# * Keys and values can contain only lowercase letters, numeric characters,
|
1542
|
+
# underscores, and dashes. All characters must use UTF-8 encoding, and
|
1543
|
+
# international characters are allowed.
|
1544
|
+
# * The key portion of a label must be unique. However, you can use the
|
1545
|
+
# same key with multiple resources.
|
1546
|
+
# * Keys must start with a lowercase letter or international character.
|
1511
1547
|
# @yield [job] a job configuration object
|
1512
1548
|
# @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
|
1513
1549
|
# configuration object for setting additional options.
|
1514
1550
|
#
|
1515
1551
|
# @return [Google::Cloud::Bigquery::ExtractJob]
|
1516
1552
|
#
|
1517
|
-
# @example
|
1553
|
+
# @example Export table data
|
1518
1554
|
# require "google/cloud/bigquery"
|
1519
1555
|
#
|
1520
1556
|
# bigquery = Google::Cloud::Bigquery.new
|
1521
1557
|
#
|
1522
1558
|
# table_id = "bigquery-public-data.samples.shakespeare"
|
1523
|
-
# extract_job = bigquery.extract_job table_id,
|
1524
|
-
# "gs://my-bucket/shakespeare.csv"
|
1559
|
+
# extract_job = bigquery.extract_job table_id, "gs://my-bucket/shakespeare.csv"
|
1525
1560
|
# extract_job.wait_until_done!
|
1526
1561
|
# extract_job.done? #=> true
|
1527
1562
|
#
|
1563
|
+
# @example Export a model
|
1564
|
+
# require "google/cloud/bigquery"
|
1565
|
+
#
|
1566
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1567
|
+
# dataset = bigquery.dataset "my_dataset"
|
1568
|
+
# model = dataset.model "my_model"
|
1569
|
+
#
|
1570
|
+
# extract_job = bigquery.extract model, "gs://my-bucket/#{model.model_id}"
|
1571
|
+
#
|
1528
1572
|
# @!group Data
|
1529
1573
|
#
|
1530
|
-
def extract_job
|
1574
|
+
def extract_job source, extract_url, format: nil, compression: nil, delimiter: nil, header: nil, job_id: nil,
|
1531
1575
|
prefix: nil, labels: nil
|
1532
1576
|
ensure_service!
|
1533
1577
|
options = { format: format, compression: compression, delimiter: delimiter, header: header, job_id: job_id,
|
1534
1578
|
prefix: prefix, labels: labels }
|
1579
|
+
source_ref = if source.respond_to? :model_ref
|
1580
|
+
source.model_ref
|
1581
|
+
else
|
1582
|
+
Service.get_table_ref source, default_ref: project_ref
|
1583
|
+
end
|
1535
1584
|
|
1536
|
-
|
1537
|
-
updater = ExtractJob::Updater.from_options service, table_ref, extract_url, options
|
1585
|
+
updater = ExtractJob::Updater.from_options service, source_ref, extract_url, options
|
1538
1586
|
|
1539
1587
|
yield updater if block_given?
|
1540
1588
|
|
@@ -1544,51 +1592,63 @@ module Google
|
|
1544
1592
|
end
|
1545
1593
|
|
1546
1594
|
##
|
1547
|
-
# Extracts the data from
|
1548
|
-
#
|
1595
|
+
# Extracts the data from a table or exports a model to Google Cloud Storage
|
1596
|
+
# using a synchronous method that blocks for a response. Timeouts
|
1549
1597
|
# and transient errors are generally handled as needed to complete the
|
1550
|
-
# job. See {#extract_job} for the asynchronous version.
|
1551
|
-
#
|
1552
|
-
#
|
1598
|
+
# job. See {#extract_job} for the asynchronous version.
|
1599
|
+
#
|
1600
|
+
# Use this method instead of {Table#extract} or {Model#extract} to
|
1601
|
+
# extract data from source tables or models in other projects.
|
1553
1602
|
#
|
1554
1603
|
# The geographic location for the job ("US", "EU", etc.) can be set via
|
1555
1604
|
# {ExtractJob::Updater#location=} in a block passed to this method.
|
1556
1605
|
#
|
1557
|
-
# @see https://cloud.google.com/bigquery/exporting-data
|
1558
|
-
# Exporting
|
1606
|
+
# @see https://cloud.google.com/bigquery/docs/exporting-data
|
1607
|
+
# Exporting table data
|
1608
|
+
# @see https://cloud.google.com/bigquery-ml/docs/exporting-models
|
1609
|
+
# Exporting models
|
1559
1610
|
#
|
1560
|
-
# @param [
|
1561
|
-
#
|
1562
|
-
# [Standard SQL Query
|
1611
|
+
# @param [Table, Model, String] source The source table or model for
|
1612
|
+
# the extract operation. This can be a table or model object; or a
|
1613
|
+
# table ID string as specified by the [Standard SQL Query
|
1563
1614
|
# Reference](https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax#from-clause)
|
1564
1615
|
# (`project-name.dataset_id.table_id`) or the [Legacy SQL Query
|
1565
1616
|
# Reference](https://cloud.google.com/bigquery/query-reference#from)
|
1566
1617
|
# (`project-name:dataset_id.table_id`).
|
1567
1618
|
# @param [Google::Cloud::Storage::File, String, Array<String>]
|
1568
1619
|
# extract_url The Google Storage file or file URI pattern(s) to which
|
1569
|
-
# BigQuery should extract
|
1570
|
-
#
|
1571
|
-
#
|
1620
|
+
# BigQuery should extract. For a model export this value should be a
|
1621
|
+
# string ending in an object name prefix, since multiple objects will
|
1622
|
+
# be exported.
|
1623
|
+
# @param [String] format The exported file format. The default value for
|
1624
|
+
# tables is `csv`. Tables with nested or repeated fields cannot be
|
1625
|
+
# exported as CSV. The default value for models is `ml_tf_saved_model`.
|
1572
1626
|
#
|
1573
|
-
#
|
1627
|
+
# Supported values for tables:
|
1574
1628
|
#
|
1575
1629
|
# * `csv` - CSV
|
1576
1630
|
# * `json` - [Newline-delimited JSON](http://jsonlines.org/)
|
1577
1631
|
# * `avro` - [Avro](http://avro.apache.org/)
|
1632
|
+
#
|
1633
|
+
# Supported values for models:
|
1634
|
+
#
|
1635
|
+
# * `ml_tf_saved_model` - TensorFlow SavedModel
|
1636
|
+
# * `ml_xgboost_booster` - XGBoost Booster
|
1578
1637
|
# @param [String] compression The compression type to use for exported
|
1579
1638
|
# files. Possible values include `GZIP` and `NONE`. The default value
|
1580
|
-
# is `NONE`.
|
1639
|
+
# is `NONE`. Not applicable when extracting models.
|
1581
1640
|
# @param [String] delimiter Delimiter to use between fields in the
|
1582
|
-
# exported data. Default is
|
1583
|
-
#
|
1584
|
-
#
|
1641
|
+
# exported table data. Default is `,`. Not applicable when extracting
|
1642
|
+
# models.
|
1643
|
+
# @param [Boolean] header Whether to print out a header row in table
|
1644
|
+
# exports. Default is `true`. Not applicable when extracting models.
|
1585
1645
|
# @yield [job] a job configuration object
|
1586
1646
|
# @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
|
1587
1647
|
# configuration object for setting additional options.
|
1588
1648
|
#
|
1589
1649
|
# @return [Boolean] Returns `true` if the extract operation succeeded.
|
1590
1650
|
#
|
1591
|
-
# @example
|
1651
|
+
# @example Export table data
|
1592
1652
|
# require "google/cloud/bigquery"
|
1593
1653
|
#
|
1594
1654
|
# bigquery = Google::Cloud::Bigquery.new
|
@@ -1596,10 +1656,19 @@ module Google
|
|
1596
1656
|
# bigquery.extract "bigquery-public-data.samples.shakespeare",
|
1597
1657
|
# "gs://my-bucket/shakespeare.csv"
|
1598
1658
|
#
|
1659
|
+
# @example Export a model
|
1660
|
+
# require "google/cloud/bigquery"
|
1661
|
+
#
|
1662
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1663
|
+
# dataset = bigquery.dataset "my_dataset"
|
1664
|
+
# model = dataset.model "my_model"
|
1665
|
+
#
|
1666
|
+
# bigquery.extract model, "gs://my-bucket/#{model.model_id}"
|
1667
|
+
#
|
1599
1668
|
# @!group Data
|
1600
1669
|
#
|
1601
|
-
def extract
|
1602
|
-
job = extract_job
|
1670
|
+
def extract source, extract_url, format: nil, compression: nil, delimiter: nil, header: nil, &block
|
1671
|
+
job = extract_job source, extract_url,
|
1603
1672
|
format: format,
|
1604
1673
|
compression: compression,
|
1605
1674
|
delimiter: delimiter,
|