gcloud 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/AUTHENTICATION.md +3 -3
- data/CHANGELOG.md +92 -0
- data/OVERVIEW.md +3 -3
- data/lib/gcloud.rb +75 -25
- data/lib/gcloud/backoff.rb +5 -1
- data/lib/gcloud/bigquery.rb +25 -43
- data/lib/gcloud/bigquery/copy_job.rb +13 -13
- data/lib/gcloud/bigquery/data.rb +20 -16
- data/lib/gcloud/bigquery/dataset.rb +202 -177
- data/lib/gcloud/bigquery/dataset/access.rb +118 -104
- data/lib/gcloud/bigquery/dataset/list.rb +14 -18
- data/lib/gcloud/bigquery/extract_job.rb +12 -12
- data/lib/gcloud/bigquery/insert_response.rb +12 -14
- data/lib/gcloud/bigquery/job.rb +45 -57
- data/lib/gcloud/bigquery/job/list.rb +18 -24
- data/lib/gcloud/bigquery/load_job.rb +35 -27
- data/lib/gcloud/bigquery/project.rb +53 -73
- data/lib/gcloud/bigquery/query_data.rb +28 -35
- data/lib/gcloud/bigquery/query_job.rb +18 -18
- data/lib/gcloud/bigquery/schema.rb +359 -0
- data/lib/gcloud/bigquery/service.rb +506 -0
- data/lib/gcloud/bigquery/table.rb +185 -266
- data/lib/gcloud/bigquery/table/list.rb +15 -19
- data/lib/gcloud/bigquery/view.rb +126 -81
- data/lib/gcloud/datastore.rb +39 -27
- data/lib/gcloud/datastore/commit.rb +2 -2
- data/lib/gcloud/datastore/dataset.rb +8 -19
- data/lib/gcloud/datastore/dataset/lookup_results.rb +2 -4
- data/lib/gcloud/datastore/dataset/query_results.rb +0 -2
- data/lib/gcloud/datastore/entity.rb +7 -1
- data/lib/gcloud/datastore/errors.rb +5 -27
- data/lib/gcloud/datastore/grpc_utils.rb +4 -3
- data/lib/gcloud/datastore/key.rb +6 -0
- data/lib/gcloud/datastore/service.rb +18 -12
- data/lib/gcloud/datastore/transaction.rb +0 -10
- data/lib/gcloud/dns.rb +29 -19
- data/lib/gcloud/dns/change.rb +10 -15
- data/lib/gcloud/dns/change/list.rb +4 -4
- data/lib/gcloud/dns/importer.rb +1 -1
- data/lib/gcloud/dns/project.rb +32 -49
- data/lib/gcloud/dns/record.rb +8 -2
- data/lib/gcloud/dns/record/list.rb +4 -4
- data/lib/gcloud/dns/service.rb +167 -0
- data/lib/gcloud/dns/zone.rb +33 -52
- data/lib/gcloud/dns/zone/list.rb +12 -16
- data/lib/gcloud/errors.rb +31 -19
- data/lib/gcloud/logging.rb +50 -39
- data/lib/gcloud/logging/entry.rb +197 -24
- data/lib/gcloud/logging/entry/list.rb +0 -2
- data/lib/gcloud/logging/logger.rb +1 -1
- data/lib/gcloud/logging/metric.rb +3 -9
- data/lib/gcloud/logging/metric/list.rb +0 -2
- data/lib/gcloud/logging/project.rb +58 -54
- data/lib/gcloud/logging/resource_descriptor.rb +2 -2
- data/lib/gcloud/logging/resource_descriptor/list.rb +0 -2
- data/lib/gcloud/logging/service.rb +32 -23
- data/lib/gcloud/logging/sink.rb +8 -14
- data/lib/gcloud/logging/sink/list.rb +0 -2
- data/lib/gcloud/pubsub.rb +21 -16
- data/lib/gcloud/pubsub/policy.rb +204 -0
- data/lib/gcloud/pubsub/project.rb +26 -38
- data/lib/gcloud/pubsub/service.rb +39 -31
- data/lib/gcloud/pubsub/subscription.rb +56 -59
- data/lib/gcloud/pubsub/subscription/list.rb +4 -4
- data/lib/gcloud/pubsub/topic.rb +69 -66
- data/lib/gcloud/pubsub/topic/list.rb +0 -2
- data/lib/gcloud/pubsub/topic/{batch.rb → publisher.rb} +15 -2
- data/lib/gcloud/resource_manager.rb +27 -26
- data/lib/gcloud/resource_manager/manager.rb +19 -39
- data/lib/gcloud/resource_manager/policy.rb +211 -0
- data/lib/gcloud/resource_manager/project.rb +97 -121
- data/lib/gcloud/resource_manager/project/list.rb +7 -7
- data/lib/gcloud/resource_manager/project/updater.rb +4 -9
- data/lib/gcloud/resource_manager/service.rb +127 -0
- data/lib/gcloud/storage.rb +24 -42
- data/lib/gcloud/storage/bucket.rb +104 -192
- data/lib/gcloud/storage/bucket/acl.rb +47 -143
- data/lib/gcloud/storage/bucket/cors.rb +55 -11
- data/lib/gcloud/storage/bucket/list.rb +14 -14
- data/lib/gcloud/storage/errors.rb +3 -43
- data/lib/gcloud/storage/file.rb +114 -111
- data/lib/gcloud/storage/file/acl.rb +27 -113
- data/lib/gcloud/storage/file/list.rb +21 -21
- data/lib/gcloud/storage/project.rb +49 -59
- data/lib/gcloud/storage/service.rb +347 -0
- data/lib/gcloud/translate.rb +24 -14
- data/lib/gcloud/translate/api.rb +12 -21
- data/lib/gcloud/translate/detection.rb +5 -5
- data/lib/gcloud/translate/language.rb +1 -1
- data/lib/gcloud/translate/service.rb +80 -0
- data/lib/gcloud/translate/translation.rb +6 -6
- data/lib/gcloud/version.rb +1 -1
- data/lib/gcloud/vision.rb +24 -15
- data/lib/gcloud/vision/annotate.rb +24 -21
- data/lib/gcloud/vision/annotation.rb +9 -9
- data/lib/gcloud/vision/annotation/entity.rb +11 -11
- data/lib/gcloud/vision/annotation/face.rb +25 -25
- data/lib/gcloud/vision/annotation/properties.rb +8 -8
- data/lib/gcloud/vision/annotation/safe_search.rb +4 -4
- data/lib/gcloud/vision/annotation/text.rb +7 -7
- data/lib/gcloud/vision/annotation/vertex.rb +1 -1
- data/lib/gcloud/vision/image.rb +11 -11
- data/lib/gcloud/vision/location.rb +5 -2
- data/lib/gcloud/vision/project.rb +14 -16
- data/lib/gcloud/vision/service.rb +66 -0
- data/lib/google/api_client.rb +0 -0
- metadata +27 -24
- data/lib/gcloud/bigquery/connection.rb +0 -624
- data/lib/gcloud/bigquery/errors.rb +0 -68
- data/lib/gcloud/bigquery/table/schema.rb +0 -234
- data/lib/gcloud/dns/connection.rb +0 -173
- data/lib/gcloud/dns/errors.rb +0 -68
- data/lib/gcloud/resource_manager/connection.rb +0 -134
- data/lib/gcloud/resource_manager/errors.rb +0 -68
- data/lib/gcloud/storage/connection.rb +0 -444
- data/lib/gcloud/translate/connection.rb +0 -85
- data/lib/gcloud/translate/errors.rb +0 -68
- data/lib/gcloud/upload.rb +0 -95
- data/lib/gcloud/vision/connection.rb +0 -63
- data/lib/gcloud/vision/errors.rb +0 -69
|
@@ -99,13 +99,16 @@ module Gcloud
|
|
|
99
99
|
##
|
|
100
100
|
# @private New Google API Client LatLng object.
|
|
101
101
|
def to_gapi
|
|
102
|
-
|
|
102
|
+
Google::Apis::VisionV1::LatLng.new(
|
|
103
|
+
latitude: latitude,
|
|
104
|
+
longitude: longitude
|
|
105
|
+
)
|
|
103
106
|
end
|
|
104
107
|
|
|
105
108
|
##
|
|
106
109
|
# @private New Location from a Google API Client LatLng object.
|
|
107
110
|
def self.from_gapi gapi
|
|
108
|
-
new gapi
|
|
111
|
+
new gapi.latitude, gapi.longitude
|
|
109
112
|
end
|
|
110
113
|
end
|
|
111
114
|
end
|
|
@@ -14,12 +14,12 @@
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
require "gcloud/gce"
|
|
17
|
-
require "gcloud/
|
|
17
|
+
require "gcloud/errors"
|
|
18
|
+
require "gcloud/vision/service"
|
|
18
19
|
require "gcloud/vision/credentials"
|
|
19
20
|
require "gcloud/vision/annotate"
|
|
20
21
|
require "gcloud/vision/image"
|
|
21
22
|
require "gcloud/vision/annotation"
|
|
22
|
-
require "gcloud/vision/errors"
|
|
23
23
|
|
|
24
24
|
module Gcloud
|
|
25
25
|
module Vision
|
|
@@ -48,15 +48,13 @@ module Gcloud
|
|
|
48
48
|
# See Gcloud#vision
|
|
49
49
|
class Project
|
|
50
50
|
##
|
|
51
|
-
# @private The
|
|
52
|
-
attr_accessor :
|
|
51
|
+
# @private The Service object.
|
|
52
|
+
attr_accessor :service
|
|
53
53
|
|
|
54
54
|
##
|
|
55
55
|
# @private Creates a new Project instance.
|
|
56
|
-
def initialize
|
|
57
|
-
|
|
58
|
-
fail ArgumentError, "project is missing" if project.empty?
|
|
59
|
-
@connection = Connection.new project, credentials
|
|
56
|
+
def initialize service
|
|
57
|
+
@service = service
|
|
60
58
|
end
|
|
61
59
|
|
|
62
60
|
# The Vision project connected to.
|
|
@@ -71,7 +69,7 @@ module Gcloud
|
|
|
71
69
|
# vision.project #=> "my-todo-project"
|
|
72
70
|
#
|
|
73
71
|
def project
|
|
74
|
-
|
|
72
|
+
service.project
|
|
75
73
|
end
|
|
76
74
|
|
|
77
75
|
##
|
|
@@ -132,7 +130,7 @@ module Gcloud
|
|
|
132
130
|
##
|
|
133
131
|
# Performs detection of Cloud Vision [features](https://cloud.google.com/vision/reference/rest/v1/images/annotate#Feature)
|
|
134
132
|
# on the given image(s). If no options for features are provided, **all**
|
|
135
|
-
# image detection features will be performed, with a default of `
|
|
133
|
+
# image detection features will be performed, with a default of `100`
|
|
136
134
|
# results for faces, landmarks, logos, and labels. If any feature option
|
|
137
135
|
# is provided, only the specified feature detections will be performed.
|
|
138
136
|
# Please review [Pricing](https://cloud.google.com/vision/docs/pricing)
|
|
@@ -257,10 +255,10 @@ module Gcloud
|
|
|
257
255
|
|
|
258
256
|
yield a if block_given?
|
|
259
257
|
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
Annotation.from_gapi
|
|
258
|
+
gapi = service.annotate a.requests
|
|
259
|
+
annotations = Array(gapi.responses).map do |g|
|
|
260
|
+
fail Error.from_error(g.error) if g.error
|
|
261
|
+
Annotation.from_gapi g
|
|
264
262
|
end
|
|
265
263
|
return annotations.first if annotations.count == 1
|
|
266
264
|
annotations
|
|
@@ -272,8 +270,8 @@ module Gcloud
|
|
|
272
270
|
|
|
273
271
|
##
|
|
274
272
|
# Raise an error unless an active connection is available.
|
|
275
|
-
def
|
|
276
|
-
fail "Must have active connection" unless
|
|
273
|
+
def ensure_service!
|
|
274
|
+
fail "Must have active connection" unless service
|
|
277
275
|
end
|
|
278
276
|
end
|
|
279
277
|
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Copyright 2016 Google Inc. All rights reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
require "gcloud/version"
|
|
17
|
+
require "gcloud/errors"
|
|
18
|
+
require "google/apis/vision_v1"
|
|
19
|
+
|
|
20
|
+
module Gcloud
|
|
21
|
+
module Vision
|
|
22
|
+
##
|
|
23
|
+
# @private
|
|
24
|
+
# Represents the service to Vision, exposing the API calls.
|
|
25
|
+
class Service
|
|
26
|
+
##
|
|
27
|
+
# Alias to the Google Client API module
|
|
28
|
+
API = Google::Apis::VisionV1
|
|
29
|
+
|
|
30
|
+
attr_accessor :project
|
|
31
|
+
attr_accessor :credentials
|
|
32
|
+
|
|
33
|
+
##
|
|
34
|
+
# Creates a new Service instance.
|
|
35
|
+
def initialize project, credentials, retries: nil, timeout: nil
|
|
36
|
+
@project = project
|
|
37
|
+
@credentials = credentials
|
|
38
|
+
@service = API::VisionService.new
|
|
39
|
+
@service.client_options.application_name = "gcloud-ruby"
|
|
40
|
+
@service.client_options.application_version = Gcloud::VERSION
|
|
41
|
+
@service.request_options.retries = retries || 3
|
|
42
|
+
@service.request_options.timeout_sec = timeout if timeout
|
|
43
|
+
@service.authorization = @credentials.client
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def service
|
|
47
|
+
return mocked_service if mocked_service
|
|
48
|
+
@service
|
|
49
|
+
end
|
|
50
|
+
attr_accessor :mocked_service
|
|
51
|
+
|
|
52
|
+
##
|
|
53
|
+
# Returns API::BatchAnnotateImagesResponse
|
|
54
|
+
def annotate requests
|
|
55
|
+
request = API::BatchAnnotateImagesRequest.new(requests: requests)
|
|
56
|
+
service.annotate_image request
|
|
57
|
+
rescue Google::Apis::Error => e
|
|
58
|
+
raise Gcloud::Error.from_error(e)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def inspect
|
|
62
|
+
"#{self.class}(#{@project})"
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
File without changes
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: gcloud
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.12.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Silvano Luciani
|
|
@@ -10,22 +10,22 @@ authors:
|
|
|
10
10
|
autorequire:
|
|
11
11
|
bindir: bin
|
|
12
12
|
cert_chain: []
|
|
13
|
-
date: 2016-
|
|
13
|
+
date: 2016-07-25 00:00:00.000000000 Z
|
|
14
14
|
dependencies:
|
|
15
15
|
- !ruby/object:Gem::Dependency
|
|
16
16
|
name: grpc
|
|
17
17
|
requirement: !ruby/object:Gem::Requirement
|
|
18
18
|
requirements:
|
|
19
|
-
- -
|
|
19
|
+
- - '='
|
|
20
20
|
- !ruby/object:Gem::Version
|
|
21
|
-
version: 0.
|
|
21
|
+
version: 1.0.0.pre1
|
|
22
22
|
type: :runtime
|
|
23
23
|
prerelease: false
|
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
|
25
25
|
requirements:
|
|
26
|
-
- -
|
|
26
|
+
- - '='
|
|
27
27
|
- !ruby/object:Gem::Version
|
|
28
|
-
version: 0.
|
|
28
|
+
version: 1.0.0.pre1
|
|
29
29
|
- !ruby/object:Gem::Dependency
|
|
30
30
|
name: google-protobuf
|
|
31
31
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -46,28 +46,34 @@ dependencies:
|
|
|
46
46
|
requirements:
|
|
47
47
|
- - ~>
|
|
48
48
|
- !ruby/object:Gem::Version
|
|
49
|
-
version: 0.
|
|
49
|
+
version: 0.9.11
|
|
50
50
|
type: :runtime
|
|
51
51
|
prerelease: false
|
|
52
52
|
version_requirements: !ruby/object:Gem::Requirement
|
|
53
53
|
requirements:
|
|
54
54
|
- - ~>
|
|
55
55
|
- !ruby/object:Gem::Version
|
|
56
|
-
version: 0.
|
|
56
|
+
version: 0.9.11
|
|
57
57
|
- !ruby/object:Gem::Dependency
|
|
58
58
|
name: mime-types
|
|
59
59
|
requirement: !ruby/object:Gem::Requirement
|
|
60
60
|
requirements:
|
|
61
|
-
- -
|
|
61
|
+
- - ! '>='
|
|
62
62
|
- !ruby/object:Gem::Version
|
|
63
63
|
version: '2.4'
|
|
64
|
+
- - <
|
|
65
|
+
- !ruby/object:Gem::Version
|
|
66
|
+
version: '4.0'
|
|
64
67
|
type: :runtime
|
|
65
68
|
prerelease: false
|
|
66
69
|
version_requirements: !ruby/object:Gem::Requirement
|
|
67
70
|
requirements:
|
|
68
|
-
- -
|
|
71
|
+
- - ! '>='
|
|
69
72
|
- !ruby/object:Gem::Version
|
|
70
73
|
version: '2.4'
|
|
74
|
+
- - <
|
|
75
|
+
- !ruby/object:Gem::Version
|
|
76
|
+
version: '4.0'
|
|
71
77
|
- !ruby/object:Gem::Dependency
|
|
72
78
|
name: digest-crc
|
|
73
79
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -282,14 +288,12 @@ files:
|
|
|
282
288
|
- lib/gcloud.rb
|
|
283
289
|
- lib/gcloud/backoff.rb
|
|
284
290
|
- lib/gcloud/bigquery.rb
|
|
285
|
-
- lib/gcloud/bigquery/connection.rb
|
|
286
291
|
- lib/gcloud/bigquery/copy_job.rb
|
|
287
292
|
- lib/gcloud/bigquery/credentials.rb
|
|
288
293
|
- lib/gcloud/bigquery/data.rb
|
|
289
294
|
- lib/gcloud/bigquery/dataset.rb
|
|
290
295
|
- lib/gcloud/bigquery/dataset/access.rb
|
|
291
296
|
- lib/gcloud/bigquery/dataset/list.rb
|
|
292
|
-
- lib/gcloud/bigquery/errors.rb
|
|
293
297
|
- lib/gcloud/bigquery/extract_job.rb
|
|
294
298
|
- lib/gcloud/bigquery/insert_response.rb
|
|
295
299
|
- lib/gcloud/bigquery/job.rb
|
|
@@ -298,9 +302,10 @@ files:
|
|
|
298
302
|
- lib/gcloud/bigquery/project.rb
|
|
299
303
|
- lib/gcloud/bigquery/query_data.rb
|
|
300
304
|
- lib/gcloud/bigquery/query_job.rb
|
|
305
|
+
- lib/gcloud/bigquery/schema.rb
|
|
306
|
+
- lib/gcloud/bigquery/service.rb
|
|
301
307
|
- lib/gcloud/bigquery/table.rb
|
|
302
308
|
- lib/gcloud/bigquery/table/list.rb
|
|
303
|
-
- lib/gcloud/bigquery/table/schema.rb
|
|
304
309
|
- lib/gcloud/bigquery/view.rb
|
|
305
310
|
- lib/gcloud/credentials.rb
|
|
306
311
|
- lib/gcloud/datastore.rb
|
|
@@ -322,13 +327,12 @@ files:
|
|
|
322
327
|
- lib/gcloud/dns.rb
|
|
323
328
|
- lib/gcloud/dns/change.rb
|
|
324
329
|
- lib/gcloud/dns/change/list.rb
|
|
325
|
-
- lib/gcloud/dns/connection.rb
|
|
326
330
|
- lib/gcloud/dns/credentials.rb
|
|
327
|
-
- lib/gcloud/dns/errors.rb
|
|
328
331
|
- lib/gcloud/dns/importer.rb
|
|
329
332
|
- lib/gcloud/dns/project.rb
|
|
330
333
|
- lib/gcloud/dns/record.rb
|
|
331
334
|
- lib/gcloud/dns/record/list.rb
|
|
335
|
+
- lib/gcloud/dns/service.rb
|
|
332
336
|
- lib/gcloud/dns/zone.rb
|
|
333
337
|
- lib/gcloud/dns/zone/list.rb
|
|
334
338
|
- lib/gcloud/dns/zone/transaction.rb
|
|
@@ -354,28 +358,28 @@ files:
|
|
|
354
358
|
- lib/gcloud/pubsub.rb
|
|
355
359
|
- lib/gcloud/pubsub/credentials.rb
|
|
356
360
|
- lib/gcloud/pubsub/message.rb
|
|
361
|
+
- lib/gcloud/pubsub/policy.rb
|
|
357
362
|
- lib/gcloud/pubsub/project.rb
|
|
358
363
|
- lib/gcloud/pubsub/received_message.rb
|
|
359
364
|
- lib/gcloud/pubsub/service.rb
|
|
360
365
|
- lib/gcloud/pubsub/subscription.rb
|
|
361
366
|
- lib/gcloud/pubsub/subscription/list.rb
|
|
362
367
|
- lib/gcloud/pubsub/topic.rb
|
|
363
|
-
- lib/gcloud/pubsub/topic/batch.rb
|
|
364
368
|
- lib/gcloud/pubsub/topic/list.rb
|
|
369
|
+
- lib/gcloud/pubsub/topic/publisher.rb
|
|
365
370
|
- lib/gcloud/resource_manager.rb
|
|
366
|
-
- lib/gcloud/resource_manager/connection.rb
|
|
367
371
|
- lib/gcloud/resource_manager/credentials.rb
|
|
368
|
-
- lib/gcloud/resource_manager/errors.rb
|
|
369
372
|
- lib/gcloud/resource_manager/manager.rb
|
|
373
|
+
- lib/gcloud/resource_manager/policy.rb
|
|
370
374
|
- lib/gcloud/resource_manager/project.rb
|
|
371
375
|
- lib/gcloud/resource_manager/project/list.rb
|
|
372
376
|
- lib/gcloud/resource_manager/project/updater.rb
|
|
377
|
+
- lib/gcloud/resource_manager/service.rb
|
|
373
378
|
- lib/gcloud/storage.rb
|
|
374
379
|
- lib/gcloud/storage/bucket.rb
|
|
375
380
|
- lib/gcloud/storage/bucket/acl.rb
|
|
376
381
|
- lib/gcloud/storage/bucket/cors.rb
|
|
377
382
|
- lib/gcloud/storage/bucket/list.rb
|
|
378
|
-
- lib/gcloud/storage/connection.rb
|
|
379
383
|
- lib/gcloud/storage/credentials.rb
|
|
380
384
|
- lib/gcloud/storage/errors.rb
|
|
381
385
|
- lib/gcloud/storage/file.rb
|
|
@@ -383,14 +387,13 @@ files:
|
|
|
383
387
|
- lib/gcloud/storage/file/list.rb
|
|
384
388
|
- lib/gcloud/storage/file/verifier.rb
|
|
385
389
|
- lib/gcloud/storage/project.rb
|
|
390
|
+
- lib/gcloud/storage/service.rb
|
|
386
391
|
- lib/gcloud/translate.rb
|
|
387
392
|
- lib/gcloud/translate/api.rb
|
|
388
|
-
- lib/gcloud/translate/connection.rb
|
|
389
393
|
- lib/gcloud/translate/detection.rb
|
|
390
|
-
- lib/gcloud/translate/errors.rb
|
|
391
394
|
- lib/gcloud/translate/language.rb
|
|
395
|
+
- lib/gcloud/translate/service.rb
|
|
392
396
|
- lib/gcloud/translate/translation.rb
|
|
393
|
-
- lib/gcloud/upload.rb
|
|
394
397
|
- lib/gcloud/version.rb
|
|
395
398
|
- lib/gcloud/vision.rb
|
|
396
399
|
- lib/gcloud/vision/annotate.rb
|
|
@@ -401,16 +404,16 @@ files:
|
|
|
401
404
|
- lib/gcloud/vision/annotation/safe_search.rb
|
|
402
405
|
- lib/gcloud/vision/annotation/text.rb
|
|
403
406
|
- lib/gcloud/vision/annotation/vertex.rb
|
|
404
|
-
- lib/gcloud/vision/connection.rb
|
|
405
407
|
- lib/gcloud/vision/credentials.rb
|
|
406
|
-
- lib/gcloud/vision/errors.rb
|
|
407
408
|
- lib/gcloud/vision/image.rb
|
|
408
409
|
- lib/gcloud/vision/location.rb
|
|
409
410
|
- lib/gcloud/vision/project.rb
|
|
411
|
+
- lib/gcloud/vision/service.rb
|
|
410
412
|
- lib/google/api/annotations.rb
|
|
411
413
|
- lib/google/api/http.rb
|
|
412
414
|
- lib/google/api/label.rb
|
|
413
415
|
- lib/google/api/monitored_resource.rb
|
|
416
|
+
- lib/google/api_client.rb
|
|
414
417
|
- lib/google/datastore/v1beta3/datastore.rb
|
|
415
418
|
- lib/google/datastore/v1beta3/datastore_services.rb
|
|
416
419
|
- lib/google/datastore/v1beta3/entity.rb
|
|
@@ -1,624 +0,0 @@
|
|
|
1
|
-
# Copyright 2015 Google Inc. All rights reserved.
|
|
2
|
-
#
|
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
-
# you may not use this file except in compliance with the License.
|
|
5
|
-
# You may obtain a copy of the License at
|
|
6
|
-
#
|
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
-
#
|
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
-
# See the License for the specific language governing permissions and
|
|
13
|
-
# limitations under the License.
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
require "pathname"
|
|
17
|
-
require "gcloud/version"
|
|
18
|
-
require "gcloud/backoff"
|
|
19
|
-
require "google/api_client"
|
|
20
|
-
require "digest/md5"
|
|
21
|
-
|
|
22
|
-
module Gcloud
|
|
23
|
-
module Bigquery
|
|
24
|
-
##
|
|
25
|
-
# @private Represents the connection to Bigquery,
|
|
26
|
-
# as well as expose the API calls.
|
|
27
|
-
class Connection
|
|
28
|
-
API_VERSION = "v2"
|
|
29
|
-
|
|
30
|
-
attr_accessor :project
|
|
31
|
-
attr_accessor :credentials
|
|
32
|
-
|
|
33
|
-
##
|
|
34
|
-
# Creates a new Connection instance.
|
|
35
|
-
def initialize project, credentials
|
|
36
|
-
@project = project
|
|
37
|
-
@credentials = credentials
|
|
38
|
-
@client = Google::APIClient.new application_name: "gcloud-ruby",
|
|
39
|
-
application_version: Gcloud::VERSION
|
|
40
|
-
@client.authorization = @credentials.client
|
|
41
|
-
@bigquery = @client.discovered_api "bigquery", API_VERSION
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
##
|
|
45
|
-
# Lists all datasets in the specified project to which you have
|
|
46
|
-
# been granted the READER dataset role.
|
|
47
|
-
def list_datasets options = {}
|
|
48
|
-
params = { projectId: @project,
|
|
49
|
-
all: options.delete(:all),
|
|
50
|
-
pageToken: options.delete(:token),
|
|
51
|
-
maxResults: options.delete(:max)
|
|
52
|
-
}.delete_if { |_, v| v.nil? }
|
|
53
|
-
|
|
54
|
-
execute(
|
|
55
|
-
api_method: @bigquery.datasets.list,
|
|
56
|
-
parameters: params
|
|
57
|
-
)
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
##
|
|
61
|
-
# Returns the dataset specified by datasetID.
|
|
62
|
-
def get_dataset dataset_id
|
|
63
|
-
execute(
|
|
64
|
-
api_method: @bigquery.datasets.get,
|
|
65
|
-
parameters: { projectId: @project, datasetId: dataset_id }
|
|
66
|
-
)
|
|
67
|
-
end
|
|
68
|
-
|
|
69
|
-
##
|
|
70
|
-
# Creates a new empty dataset.
|
|
71
|
-
def insert_dataset dataset_id, options = {}
|
|
72
|
-
execute(
|
|
73
|
-
api_method: @bigquery.datasets.insert,
|
|
74
|
-
parameters: { projectId: @project },
|
|
75
|
-
body_object: insert_dataset_request(dataset_id, options)
|
|
76
|
-
)
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
##
|
|
80
|
-
# Updates information in an existing dataset, only replacing
|
|
81
|
-
# fields that are provided in the submitted dataset resource.
|
|
82
|
-
def patch_dataset dataset_id, options = {}
|
|
83
|
-
project_id = options[:project_id] || @project
|
|
84
|
-
|
|
85
|
-
execute(
|
|
86
|
-
api_method: @bigquery.datasets.patch,
|
|
87
|
-
parameters: { projectId: project_id, datasetId: dataset_id },
|
|
88
|
-
body_object: patch_dataset_request(options)
|
|
89
|
-
)
|
|
90
|
-
end
|
|
91
|
-
|
|
92
|
-
##
|
|
93
|
-
# Deletes the dataset specified by the datasetId value.
|
|
94
|
-
# Before you can delete a dataset, you must delete all its tables,
|
|
95
|
-
# either manually or by specifying force: true in options.
|
|
96
|
-
# Immediately after deletion, you can create another dataset with
|
|
97
|
-
# the same name.
|
|
98
|
-
def delete_dataset dataset_id, force = nil
|
|
99
|
-
execute(
|
|
100
|
-
api_method: @bigquery.datasets.delete,
|
|
101
|
-
parameters: { projectId: @project, datasetId: dataset_id,
|
|
102
|
-
deleteContents: force
|
|
103
|
-
}.delete_if { |_, v| v.nil? }
|
|
104
|
-
)
|
|
105
|
-
end
|
|
106
|
-
|
|
107
|
-
##
|
|
108
|
-
# Lists all tables in the specified dataset.
|
|
109
|
-
# Requires the READER dataset role.
|
|
110
|
-
def list_tables dataset_id, options = {}
|
|
111
|
-
params = { projectId: @project,
|
|
112
|
-
datasetId: dataset_id,
|
|
113
|
-
pageToken: options.delete(:token),
|
|
114
|
-
maxResults: options.delete(:max)
|
|
115
|
-
}.delete_if { |_, v| v.nil? }
|
|
116
|
-
|
|
117
|
-
execute(
|
|
118
|
-
api_method: @bigquery.tables.list,
|
|
119
|
-
parameters: params
|
|
120
|
-
)
|
|
121
|
-
end
|
|
122
|
-
|
|
123
|
-
def get_project_table project_id, dataset_id, table_id
|
|
124
|
-
execute(
|
|
125
|
-
api_method: @bigquery.tables.get,
|
|
126
|
-
parameters: { projectId: project_id, datasetId: dataset_id,
|
|
127
|
-
tableId: table_id }
|
|
128
|
-
)
|
|
129
|
-
end
|
|
130
|
-
|
|
131
|
-
##
|
|
132
|
-
# Gets the specified table resource by table ID.
|
|
133
|
-
# This method does not return the data in the table,
|
|
134
|
-
# it only returns the table resource,
|
|
135
|
-
# which describes the structure of this table.
|
|
136
|
-
def get_table dataset_id, table_id
|
|
137
|
-
get_project_table @project, dataset_id, table_id
|
|
138
|
-
end
|
|
139
|
-
|
|
140
|
-
##
|
|
141
|
-
# Creates a new, empty table in the dataset.
|
|
142
|
-
def insert_table dataset_id, table_id, options = {}
|
|
143
|
-
execute(
|
|
144
|
-
api_method: @bigquery.tables.insert,
|
|
145
|
-
parameters: { projectId: @project, datasetId: dataset_id },
|
|
146
|
-
body_object: insert_table_request(dataset_id, table_id, options)
|
|
147
|
-
)
|
|
148
|
-
end
|
|
149
|
-
|
|
150
|
-
##
|
|
151
|
-
# Updates information in an existing table, replacing fields that
|
|
152
|
-
# are provided in the submitted table resource.
|
|
153
|
-
def patch_table dataset_id, table_id, options = {}
|
|
154
|
-
execute(
|
|
155
|
-
api_method: @bigquery.tables.patch,
|
|
156
|
-
parameters: { projectId: @project, datasetId: dataset_id,
|
|
157
|
-
tableId: table_id },
|
|
158
|
-
body_object: patch_table_request(options)
|
|
159
|
-
)
|
|
160
|
-
end
|
|
161
|
-
|
|
162
|
-
##
|
|
163
|
-
# Deletes the table specified by tableId from the dataset.
|
|
164
|
-
# If the table contains data, all the data will be deleted.
|
|
165
|
-
def delete_table dataset_id, table_id
|
|
166
|
-
execute(
|
|
167
|
-
api_method: @bigquery.tables.delete,
|
|
168
|
-
parameters: { projectId: @project, datasetId: dataset_id,
|
|
169
|
-
tableId: table_id }
|
|
170
|
-
)
|
|
171
|
-
end
|
|
172
|
-
|
|
173
|
-
##
|
|
174
|
-
# Retrieves data from the table.
|
|
175
|
-
def list_tabledata dataset_id, table_id, options = {}
|
|
176
|
-
params = { projectId: @project,
|
|
177
|
-
datasetId: dataset_id, tableId: table_id,
|
|
178
|
-
pageToken: options.delete(:token),
|
|
179
|
-
maxResults: options.delete(:max),
|
|
180
|
-
startIndex: options.delete(:start)
|
|
181
|
-
}.delete_if { |_, v| v.nil? }
|
|
182
|
-
|
|
183
|
-
execute(
|
|
184
|
-
api_method: @bigquery.tabledata.list,
|
|
185
|
-
parameters: params
|
|
186
|
-
)
|
|
187
|
-
end
|
|
188
|
-
|
|
189
|
-
def insert_tabledata dataset_id, table_id, rows, options = {}
|
|
190
|
-
execute(
|
|
191
|
-
api_method: @bigquery.tabledata.insert_all,
|
|
192
|
-
parameters: { projectId: @project,
|
|
193
|
-
datasetId: dataset_id,
|
|
194
|
-
tableId: table_id },
|
|
195
|
-
body_object: insert_tabledata_rows(rows, options)
|
|
196
|
-
)
|
|
197
|
-
end
|
|
198
|
-
|
|
199
|
-
##
|
|
200
|
-
# Lists all jobs in the specified project to which you have
|
|
201
|
-
# been granted the READER job role.
|
|
202
|
-
def list_jobs options = {}
|
|
203
|
-
execute(
|
|
204
|
-
api_method: @bigquery.jobs.list,
|
|
205
|
-
parameters: list_jobs_params(options)
|
|
206
|
-
)
|
|
207
|
-
end
|
|
208
|
-
|
|
209
|
-
##
|
|
210
|
-
# Returns the job specified by jobID.
|
|
211
|
-
def get_job job_id
|
|
212
|
-
execute(
|
|
213
|
-
api_method: @bigquery.jobs.get,
|
|
214
|
-
parameters: { projectId: @project, jobId: job_id }
|
|
215
|
-
)
|
|
216
|
-
end
|
|
217
|
-
|
|
218
|
-
def insert_job config
|
|
219
|
-
execute(
|
|
220
|
-
api_method: @bigquery.jobs.insert,
|
|
221
|
-
parameters: { projectId: @project },
|
|
222
|
-
body_object: { "configuration" => config }
|
|
223
|
-
)
|
|
224
|
-
end
|
|
225
|
-
|
|
226
|
-
def query_job query, options = {}
|
|
227
|
-
execute(
|
|
228
|
-
api_method: @bigquery.jobs.insert,
|
|
229
|
-
parameters: { projectId: @project },
|
|
230
|
-
body_object: query_table_config(query, options)
|
|
231
|
-
)
|
|
232
|
-
end
|
|
233
|
-
|
|
234
|
-
def query query, options = {}
|
|
235
|
-
execute(
|
|
236
|
-
api_method: @bigquery.jobs.query,
|
|
237
|
-
parameters: { projectId: @project },
|
|
238
|
-
body_object: query_config(query, options)
|
|
239
|
-
)
|
|
240
|
-
end
|
|
241
|
-
|
|
242
|
-
##
|
|
243
|
-
# Returns the query data for the job
|
|
244
|
-
def job_query_results job_id, options = {}
|
|
245
|
-
params = { projectId: @project, jobId: job_id,
|
|
246
|
-
pageToken: options.delete(:token),
|
|
247
|
-
maxResults: options.delete(:max),
|
|
248
|
-
startIndex: options.delete(:start),
|
|
249
|
-
timeoutMs: options.delete(:timeout)
|
|
250
|
-
}.delete_if { |_, v| v.nil? }
|
|
251
|
-
|
|
252
|
-
execute(
|
|
253
|
-
api_method: @bigquery.jobs.get_query_results,
|
|
254
|
-
parameters: params
|
|
255
|
-
)
|
|
256
|
-
end
|
|
257
|
-
|
|
258
|
-
def copy_table source, target, options = {}
|
|
259
|
-
execute(
|
|
260
|
-
api_method: @bigquery.jobs.insert,
|
|
261
|
-
parameters: { projectId: @project },
|
|
262
|
-
body_object: copy_table_config(source, target, options)
|
|
263
|
-
)
|
|
264
|
-
end
|
|
265
|
-
|
|
266
|
-
def link_table table, urls, options = {}
|
|
267
|
-
execute(
|
|
268
|
-
api_method: @bigquery.jobs.insert,
|
|
269
|
-
parameters: { projectId: @project },
|
|
270
|
-
body_object: link_table_config(table, urls, options)
|
|
271
|
-
)
|
|
272
|
-
end
|
|
273
|
-
|
|
274
|
-
def extract_table table, storage_files, options = {}
|
|
275
|
-
execute(
|
|
276
|
-
api_method: @bigquery.jobs.insert,
|
|
277
|
-
parameters: { projectId: @project },
|
|
278
|
-
body_object: extract_table_config(table, storage_files, options)
|
|
279
|
-
)
|
|
280
|
-
end
|
|
281
|
-
|
|
282
|
-
def load_table table, storage_url, options = {}
|
|
283
|
-
execute(
|
|
284
|
-
api_method: @bigquery.jobs.insert,
|
|
285
|
-
parameters: { projectId: @project },
|
|
286
|
-
body_object: load_table_config(table, storage_url,
|
|
287
|
-
Array(storage_url).first, options)
|
|
288
|
-
)
|
|
289
|
-
end
|
|
290
|
-
|
|
291
|
-
def load_multipart table, file, options = {}
|
|
292
|
-
media = load_media file
|
|
293
|
-
|
|
294
|
-
execute(
|
|
295
|
-
api_method: @bigquery.jobs.insert,
|
|
296
|
-
media: media,
|
|
297
|
-
parameters: { projectId: @project, uploadType: "multipart" },
|
|
298
|
-
body_object: load_table_config(table, nil, file, options)
|
|
299
|
-
)
|
|
300
|
-
end
|
|
301
|
-
|
|
302
|
-
def load_resumable table, file, chunk_size = nil, options = {}
|
|
303
|
-
media = load_media file, chunk_size
|
|
304
|
-
|
|
305
|
-
result = execute(
|
|
306
|
-
api_method: @bigquery.jobs.insert,
|
|
307
|
-
media: media,
|
|
308
|
-
parameters: { projectId: @project, uploadType: "resumable" },
|
|
309
|
-
body_object: load_table_config(table, nil, file, options)
|
|
310
|
-
)
|
|
311
|
-
upload = result.resumable_upload
|
|
312
|
-
result = execute upload while upload.resumable?
|
|
313
|
-
result
|
|
314
|
-
end
|
|
315
|
-
|
|
316
|
-
def default_access_rules
|
|
317
|
-
[
|
|
318
|
-
{ "role" => "OWNER", "specialGroup" => "projectOwners" },
|
|
319
|
-
{ "role" => "WRITER", "specialGroup" => "projectWriters" },
|
|
320
|
-
{ "role" => "READER", "specialGroup" => "projectReaders" },
|
|
321
|
-
{ "role" => "OWNER", "userByEmail" => credentials.issuer }
|
|
322
|
-
]
|
|
323
|
-
end
|
|
324
|
-
|
|
325
|
-
##
|
|
326
|
-
# Extracts at least `tbl` group, and possibly `dts` and `prj` groups,
|
|
327
|
-
# from strings in the formats: "my_table", "my_dataset.my_table", or
|
|
328
|
-
# "my-project:my_dataset.my_table". Then merges project_id and
|
|
329
|
-
# dataset_id from the default table if they are missing.
|
|
330
|
-
def self.table_ref_from_s str, default_table_ref
|
|
331
|
-
str = str.to_s
|
|
332
|
-
m = /\A(((?<prj>\S*):)?(?<dts>\S*)\.)?(?<tbl>\S*)\z/.match str
|
|
333
|
-
unless m
|
|
334
|
-
fail ArgumentError, "unable to identify table from #{str.inspect}"
|
|
335
|
-
end
|
|
336
|
-
str_table_ref = { "projectId" => m["prj"],
|
|
337
|
-
"datasetId" => m["dts"],
|
|
338
|
-
"tableId" => m["tbl"] }.delete_if { |_, v| v.nil? }
|
|
339
|
-
default_table_ref.merge str_table_ref
|
|
340
|
-
end
|
|
341
|
-
|
|
342
|
-
def inspect
|
|
343
|
-
"#{self.class}(#{@project})"
|
|
344
|
-
end
|
|
345
|
-
|
|
346
|
-
protected
|
|
347
|
-
|
|
348
|
-
##
|
|
349
|
-
# Create the HTTP body for insert dataset
|
|
350
|
-
def insert_dataset_request dataset_id, options = {}
|
|
351
|
-
{
|
|
352
|
-
"kind" => "bigquery#dataset",
|
|
353
|
-
"datasetReference" => {
|
|
354
|
-
"projectId" => @project,
|
|
355
|
-
"datasetId" => dataset_id },
|
|
356
|
-
"friendlyName" => options[:name],
|
|
357
|
-
"description" => options[:description],
|
|
358
|
-
"defaultTableExpirationMs" => options[:expiration],
|
|
359
|
-
"access" => options[:access],
|
|
360
|
-
"location" => options[:location]
|
|
361
|
-
}.delete_if { |_, v| v.nil? }
|
|
362
|
-
end
|
|
363
|
-
|
|
364
|
-
def patch_dataset_request options = {}
|
|
365
|
-
{
|
|
366
|
-
friendlyName: options[:name],
|
|
367
|
-
description: options[:description],
|
|
368
|
-
defaultTableExpirationMs: options[:default_expiration],
|
|
369
|
-
access: options[:access]
|
|
370
|
-
}.delete_if { |_, v| v.nil? }
|
|
371
|
-
end
|
|
372
|
-
|
|
373
|
-
##
|
|
374
|
-
# The parameters for the list_jobs call.
|
|
375
|
-
def list_jobs_params options = {}
|
|
376
|
-
params = { projectId: @project,
|
|
377
|
-
allUsers: options.delete(:all),
|
|
378
|
-
pageToken: options.delete(:token),
|
|
379
|
-
maxResults: options.delete(:max),
|
|
380
|
-
stateFilter: options.delete(:filter),
|
|
381
|
-
projection: "full"
|
|
382
|
-
}.delete_if { |_, v| v.nil? }
|
|
383
|
-
params
|
|
384
|
-
end
|
|
385
|
-
|
|
386
|
-
##
|
|
387
|
-
# Create the HTTP body for insert table
|
|
388
|
-
def insert_table_request dataset_id, table_id, options = {}
|
|
389
|
-
hash = {
|
|
390
|
-
tableReference: {
|
|
391
|
-
projectId: @project, datasetId: dataset_id, tableId: table_id
|
|
392
|
-
},
|
|
393
|
-
friendlyName: options[:name],
|
|
394
|
-
description: options[:description],
|
|
395
|
-
schema: options[:schema]
|
|
396
|
-
}.delete_if { |_, v| v.nil? }
|
|
397
|
-
hash["view"] = { "query" => options[:query] } if options[:query]
|
|
398
|
-
hash
|
|
399
|
-
end
|
|
400
|
-
|
|
401
|
-
def patch_table_request options = {}
|
|
402
|
-
body = { friendlyName: options[:name],
|
|
403
|
-
description: options[:description],
|
|
404
|
-
schema: options[:schema]
|
|
405
|
-
}.delete_if { |_, v| v.nil? }
|
|
406
|
-
body["view"] = { "query" => options[:query] } if options[:query]
|
|
407
|
-
body
|
|
408
|
-
end
|
|
409
|
-
|
|
410
|
-
def insert_tabledata_rows rows, options = {}
|
|
411
|
-
{
|
|
412
|
-
"kind" => "bigquery#tableDataInsertAllRequest",
|
|
413
|
-
"skipInvalidRows" => options[:skip_invalid],
|
|
414
|
-
"ignoreUnknownValues" => options[:ignore_unknown],
|
|
415
|
-
"rows" => rows.map do |row|
|
|
416
|
-
{ "insertId" => Digest::MD5.base64digest(row.inspect),
|
|
417
|
-
"json" => row }
|
|
418
|
-
end
|
|
419
|
-
}.delete_if { |_, v| v.nil? }
|
|
420
|
-
end
|
|
421
|
-
|
|
422
|
-
# rubocop:disable all
|
|
423
|
-
# Disabled rubocop because the API is verbose and so these methods
|
|
424
|
-
# are going to be verbose.
|
|
425
|
-
|
|
426
|
-
##
|
|
427
|
-
# Job description for query job
|
|
428
|
-
def query_table_config query, options
|
|
429
|
-
dest_table = nil
|
|
430
|
-
if options[:table]
|
|
431
|
-
dest_table = { "projectId" => options[:table].project_id,
|
|
432
|
-
"datasetId" => options[:table].dataset_id,
|
|
433
|
-
"tableId" => options[:table].table_id }
|
|
434
|
-
end
|
|
435
|
-
default_dataset = nil
|
|
436
|
-
if dataset = options[:dataset]
|
|
437
|
-
if dataset.respond_to? :dataset_id
|
|
438
|
-
default_dataset = { "projectId" => dataset.project_id,
|
|
439
|
-
"datasetId" => dataset.dataset_id }
|
|
440
|
-
else
|
|
441
|
-
default_dataset = { "datasetId" => dataset }
|
|
442
|
-
end
|
|
443
|
-
end
|
|
444
|
-
{
|
|
445
|
-
"configuration" => {
|
|
446
|
-
"query" => {
|
|
447
|
-
"query" => query,
|
|
448
|
-
# "tableDefinitions" => { ... },
|
|
449
|
-
"priority" => priority_value(options[:priority]),
|
|
450
|
-
"useQueryCache" => options[:cache],
|
|
451
|
-
"destinationTable" => dest_table,
|
|
452
|
-
"createDisposition" => create_disposition(options[:create]),
|
|
453
|
-
"writeDisposition" => write_disposition(options[:write]),
|
|
454
|
-
"allowLargeResults" => options[:large_results],
|
|
455
|
-
"flattenResults" => options[:flatten],
|
|
456
|
-
"defaultDataset" => default_dataset
|
|
457
|
-
}.delete_if { |_, v| v.nil? }
|
|
458
|
-
}.delete_if { |_, v| v.nil? }
|
|
459
|
-
}
|
|
460
|
-
end
|
|
461
|
-
|
|
462
|
-
def query_config query, options = {}
|
|
463
|
-
dataset_config = nil
|
|
464
|
-
dataset_config = {
|
|
465
|
-
"datasetId" => options[:dataset],
|
|
466
|
-
"projectId" => options[:project] || @project
|
|
467
|
-
} if options[:dataset]
|
|
468
|
-
|
|
469
|
-
{
|
|
470
|
-
"kind" => "bigquery#queryRequest",
|
|
471
|
-
"query" => query,
|
|
472
|
-
"maxResults" => options[:max],
|
|
473
|
-
"defaultDataset" => dataset_config,
|
|
474
|
-
"timeoutMs" => options[:timeout],
|
|
475
|
-
"dryRun" => options[:dryrun],
|
|
476
|
-
"useQueryCache" => options[:cache]
|
|
477
|
-
}.delete_if { |_, v| v.nil? }
|
|
478
|
-
end
|
|
479
|
-
|
|
480
|
-
##
|
|
481
|
-
# Job description for copy job
|
|
482
|
-
def copy_table_config source, target, options = {}
|
|
483
|
-
{
|
|
484
|
-
"configuration" => {
|
|
485
|
-
"copy" => {
|
|
486
|
-
"sourceTable" => source,
|
|
487
|
-
"destinationTable" => target,
|
|
488
|
-
"createDisposition" => create_disposition(options[:create]),
|
|
489
|
-
"writeDisposition" => write_disposition(options[:write])
|
|
490
|
-
}.delete_if { |_, v| v.nil? },
|
|
491
|
-
"dryRun" => options[:dryrun]
|
|
492
|
-
}.delete_if { |_, v| v.nil? }
|
|
493
|
-
}
|
|
494
|
-
end
|
|
495
|
-
|
|
496
|
-
def link_table_config table, urls, options = {}
|
|
497
|
-
path = Array(urls).first
|
|
498
|
-
{
|
|
499
|
-
"configuration" => {
|
|
500
|
-
"link" => {
|
|
501
|
-
"sourceUri" => Array(urls),
|
|
502
|
-
"destinationTable" => table,
|
|
503
|
-
"createDisposition" => create_disposition(options[:create]),
|
|
504
|
-
"writeDisposition" => write_disposition(options[:write]),
|
|
505
|
-
"sourceFormat" => source_format(path, options[:format])
|
|
506
|
-
}.delete_if { |_, v| v.nil? },
|
|
507
|
-
"dryRun" => options[:dryrun]
|
|
508
|
-
}.delete_if { |_, v| v.nil? }
|
|
509
|
-
}
|
|
510
|
-
end
|
|
511
|
-
|
|
512
|
-
def extract_table_config table, storage_files, options = {}
|
|
513
|
-
storage_urls = Array(storage_files).map do |url|
|
|
514
|
-
url.respond_to?(:to_gs_url) ? url.to_gs_url : url
|
|
515
|
-
end
|
|
516
|
-
dest_format = source_format storage_urls.first, options[:format]
|
|
517
|
-
{
|
|
518
|
-
"configuration" => {
|
|
519
|
-
"extract" => {
|
|
520
|
-
"destinationUris" => Array(storage_urls),
|
|
521
|
-
"sourceTable" => table,
|
|
522
|
-
"destinationFormat" => dest_format,
|
|
523
|
-
"compression" => options[:compression],
|
|
524
|
-
"fieldDelimiter" => options[:delimiter],
|
|
525
|
-
"printHeader" => options[:header]
|
|
526
|
-
}.delete_if { |_, v| v.nil? },
|
|
527
|
-
"dryRun" => options[:dryrun]
|
|
528
|
-
}.delete_if { |_, v| v.nil? }
|
|
529
|
-
}
|
|
530
|
-
end
|
|
531
|
-
|
|
532
|
-
def load_table_config table, urls, file, options = {}
|
|
533
|
-
path = Array(urls).first
|
|
534
|
-
path = Pathname(file).to_path unless file.nil?
|
|
535
|
-
{
|
|
536
|
-
"configuration" => {
|
|
537
|
-
"load" => {
|
|
538
|
-
"sourceUris" => Array(urls),
|
|
539
|
-
"destinationTable" => table,
|
|
540
|
-
"createDisposition" => create_disposition(options[:create]),
|
|
541
|
-
"writeDisposition" => write_disposition(options[:write]),
|
|
542
|
-
"sourceFormat" => source_format(path, options[:format]),
|
|
543
|
-
"projectionFields" => projection_fields(options[:projection_fields]),
|
|
544
|
-
"allowJaggedRows" => options[:jagged_rows],
|
|
545
|
-
"allowQuotedNewlines" => options[:quoted_newlines],
|
|
546
|
-
"encoding" => options[:encoding],
|
|
547
|
-
"fieldDelimiter" => options[:delimiter],
|
|
548
|
-
"ignoreUnknownValues" => options[:ignore_unknown],
|
|
549
|
-
"maxBadRecords" => options[:max_bad_records],
|
|
550
|
-
"quote" => options[:quote],
|
|
551
|
-
"schema" => options[:schema],
|
|
552
|
-
"skipLeadingRows" => options[:skip_leading]
|
|
553
|
-
}.delete_if { |_, v| v.nil? },
|
|
554
|
-
"dryRun" => options[:dryrun]
|
|
555
|
-
}.delete_if { |_, v| v.nil? }
|
|
556
|
-
}
|
|
557
|
-
end
|
|
558
|
-
|
|
559
|
-
def create_disposition str
|
|
560
|
-
{ "create_if_needed" => "CREATE_IF_NEEDED",
|
|
561
|
-
"createifneeded" => "CREATE_IF_NEEDED",
|
|
562
|
-
"if_needed" => "CREATE_IF_NEEDED",
|
|
563
|
-
"needed" => "CREATE_IF_NEEDED",
|
|
564
|
-
"create_never" => "CREATE_NEVER",
|
|
565
|
-
"createnever" => "CREATE_NEVER",
|
|
566
|
-
"never" => "CREATE_NEVER" }[str.to_s.downcase]
|
|
567
|
-
end
|
|
568
|
-
|
|
569
|
-
def write_disposition str
|
|
570
|
-
{ "write_truncate" => "WRITE_TRUNCATE",
|
|
571
|
-
"writetruncate" => "WRITE_TRUNCATE",
|
|
572
|
-
"truncate" => "WRITE_TRUNCATE",
|
|
573
|
-
"write_append" => "WRITE_APPEND",
|
|
574
|
-
"writeappend" => "WRITE_APPEND",
|
|
575
|
-
"append" => "WRITE_APPEND",
|
|
576
|
-
"write_empty" => "WRITE_EMPTY",
|
|
577
|
-
"writeempty" => "WRITE_EMPTY",
|
|
578
|
-
"empty" => "WRITE_EMPTY" }[str.to_s.downcase]
|
|
579
|
-
end
|
|
580
|
-
|
|
581
|
-
def priority_value str
|
|
582
|
-
{ "batch" => "BATCH",
|
|
583
|
-
"interactive" => "INTERACTIVE" }[str.to_s.downcase]
|
|
584
|
-
end
|
|
585
|
-
|
|
586
|
-
def source_format path, format
|
|
587
|
-
val = { "csv" => "CSV",
|
|
588
|
-
"json" => "NEWLINE_DELIMITED_JSON",
|
|
589
|
-
"newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
|
|
590
|
-
"avro" => "AVRO",
|
|
591
|
-
"datastore" => "DATASTORE_BACKUP",
|
|
592
|
-
"datastore_backup" => "DATASTORE_BACKUP"}[format.to_s.downcase]
|
|
593
|
-
return val unless val.nil?
|
|
594
|
-
return nil if path.nil?
|
|
595
|
-
return "CSV" if path.end_with? ".csv"
|
|
596
|
-
return "NEWLINE_DELIMITED_JSON" if path.end_with? ".json"
|
|
597
|
-
return "AVRO" if path.end_with? ".avro"
|
|
598
|
-
return "DATASTORE_BACKUP" if path.end_with? ".backup_info"
|
|
599
|
-
nil
|
|
600
|
-
end
|
|
601
|
-
|
|
602
|
-
def projection_fields array_or_str
|
|
603
|
-
Array(array_or_str) unless array_or_str.nil?
|
|
604
|
-
end
|
|
605
|
-
|
|
606
|
-
# rubocop:enable all
|
|
607
|
-
|
|
608
|
-
def load_media file, chunk_size = nil
|
|
609
|
-
local_path = Pathname(file).to_path
|
|
610
|
-
mime_type = "application/octet-stream"
|
|
611
|
-
|
|
612
|
-
media = Google::APIClient::UploadIO.new local_path, mime_type
|
|
613
|
-
media.chunk_size = chunk_size unless chunk_size.nil?
|
|
614
|
-
media
|
|
615
|
-
end
|
|
616
|
-
|
|
617
|
-
def execute options
|
|
618
|
-
Gcloud::Backoff.new.execute_gapi do
|
|
619
|
-
@client.execute options
|
|
620
|
-
end
|
|
621
|
-
end
|
|
622
|
-
end
|
|
623
|
-
end
|
|
624
|
-
end
|