gcloud 0.11.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. checksums.yaml +8 -8
  2. data/AUTHENTICATION.md +3 -3
  3. data/CHANGELOG.md +92 -0
  4. data/OVERVIEW.md +3 -3
  5. data/lib/gcloud.rb +75 -25
  6. data/lib/gcloud/backoff.rb +5 -1
  7. data/lib/gcloud/bigquery.rb +25 -43
  8. data/lib/gcloud/bigquery/copy_job.rb +13 -13
  9. data/lib/gcloud/bigquery/data.rb +20 -16
  10. data/lib/gcloud/bigquery/dataset.rb +202 -177
  11. data/lib/gcloud/bigquery/dataset/access.rb +118 -104
  12. data/lib/gcloud/bigquery/dataset/list.rb +14 -18
  13. data/lib/gcloud/bigquery/extract_job.rb +12 -12
  14. data/lib/gcloud/bigquery/insert_response.rb +12 -14
  15. data/lib/gcloud/bigquery/job.rb +45 -57
  16. data/lib/gcloud/bigquery/job/list.rb +18 -24
  17. data/lib/gcloud/bigquery/load_job.rb +35 -27
  18. data/lib/gcloud/bigquery/project.rb +53 -73
  19. data/lib/gcloud/bigquery/query_data.rb +28 -35
  20. data/lib/gcloud/bigquery/query_job.rb +18 -18
  21. data/lib/gcloud/bigquery/schema.rb +359 -0
  22. data/lib/gcloud/bigquery/service.rb +506 -0
  23. data/lib/gcloud/bigquery/table.rb +185 -266
  24. data/lib/gcloud/bigquery/table/list.rb +15 -19
  25. data/lib/gcloud/bigquery/view.rb +126 -81
  26. data/lib/gcloud/datastore.rb +39 -27
  27. data/lib/gcloud/datastore/commit.rb +2 -2
  28. data/lib/gcloud/datastore/dataset.rb +8 -19
  29. data/lib/gcloud/datastore/dataset/lookup_results.rb +2 -4
  30. data/lib/gcloud/datastore/dataset/query_results.rb +0 -2
  31. data/lib/gcloud/datastore/entity.rb +7 -1
  32. data/lib/gcloud/datastore/errors.rb +5 -27
  33. data/lib/gcloud/datastore/grpc_utils.rb +4 -3
  34. data/lib/gcloud/datastore/key.rb +6 -0
  35. data/lib/gcloud/datastore/service.rb +18 -12
  36. data/lib/gcloud/datastore/transaction.rb +0 -10
  37. data/lib/gcloud/dns.rb +29 -19
  38. data/lib/gcloud/dns/change.rb +10 -15
  39. data/lib/gcloud/dns/change/list.rb +4 -4
  40. data/lib/gcloud/dns/importer.rb +1 -1
  41. data/lib/gcloud/dns/project.rb +32 -49
  42. data/lib/gcloud/dns/record.rb +8 -2
  43. data/lib/gcloud/dns/record/list.rb +4 -4
  44. data/lib/gcloud/dns/service.rb +167 -0
  45. data/lib/gcloud/dns/zone.rb +33 -52
  46. data/lib/gcloud/dns/zone/list.rb +12 -16
  47. data/lib/gcloud/errors.rb +31 -19
  48. data/lib/gcloud/logging.rb +50 -39
  49. data/lib/gcloud/logging/entry.rb +197 -24
  50. data/lib/gcloud/logging/entry/list.rb +0 -2
  51. data/lib/gcloud/logging/logger.rb +1 -1
  52. data/lib/gcloud/logging/metric.rb +3 -9
  53. data/lib/gcloud/logging/metric/list.rb +0 -2
  54. data/lib/gcloud/logging/project.rb +58 -54
  55. data/lib/gcloud/logging/resource_descriptor.rb +2 -2
  56. data/lib/gcloud/logging/resource_descriptor/list.rb +0 -2
  57. data/lib/gcloud/logging/service.rb +32 -23
  58. data/lib/gcloud/logging/sink.rb +8 -14
  59. data/lib/gcloud/logging/sink/list.rb +0 -2
  60. data/lib/gcloud/pubsub.rb +21 -16
  61. data/lib/gcloud/pubsub/policy.rb +204 -0
  62. data/lib/gcloud/pubsub/project.rb +26 -38
  63. data/lib/gcloud/pubsub/service.rb +39 -31
  64. data/lib/gcloud/pubsub/subscription.rb +56 -59
  65. data/lib/gcloud/pubsub/subscription/list.rb +4 -4
  66. data/lib/gcloud/pubsub/topic.rb +69 -66
  67. data/lib/gcloud/pubsub/topic/list.rb +0 -2
  68. data/lib/gcloud/pubsub/topic/{batch.rb → publisher.rb} +15 -2
  69. data/lib/gcloud/resource_manager.rb +27 -26
  70. data/lib/gcloud/resource_manager/manager.rb +19 -39
  71. data/lib/gcloud/resource_manager/policy.rb +211 -0
  72. data/lib/gcloud/resource_manager/project.rb +97 -121
  73. data/lib/gcloud/resource_manager/project/list.rb +7 -7
  74. data/lib/gcloud/resource_manager/project/updater.rb +4 -9
  75. data/lib/gcloud/resource_manager/service.rb +127 -0
  76. data/lib/gcloud/storage.rb +24 -42
  77. data/lib/gcloud/storage/bucket.rb +104 -192
  78. data/lib/gcloud/storage/bucket/acl.rb +47 -143
  79. data/lib/gcloud/storage/bucket/cors.rb +55 -11
  80. data/lib/gcloud/storage/bucket/list.rb +14 -14
  81. data/lib/gcloud/storage/errors.rb +3 -43
  82. data/lib/gcloud/storage/file.rb +114 -111
  83. data/lib/gcloud/storage/file/acl.rb +27 -113
  84. data/lib/gcloud/storage/file/list.rb +21 -21
  85. data/lib/gcloud/storage/project.rb +49 -59
  86. data/lib/gcloud/storage/service.rb +347 -0
  87. data/lib/gcloud/translate.rb +24 -14
  88. data/lib/gcloud/translate/api.rb +12 -21
  89. data/lib/gcloud/translate/detection.rb +5 -5
  90. data/lib/gcloud/translate/language.rb +1 -1
  91. data/lib/gcloud/translate/service.rb +80 -0
  92. data/lib/gcloud/translate/translation.rb +6 -6
  93. data/lib/gcloud/version.rb +1 -1
  94. data/lib/gcloud/vision.rb +24 -15
  95. data/lib/gcloud/vision/annotate.rb +24 -21
  96. data/lib/gcloud/vision/annotation.rb +9 -9
  97. data/lib/gcloud/vision/annotation/entity.rb +11 -11
  98. data/lib/gcloud/vision/annotation/face.rb +25 -25
  99. data/lib/gcloud/vision/annotation/properties.rb +8 -8
  100. data/lib/gcloud/vision/annotation/safe_search.rb +4 -4
  101. data/lib/gcloud/vision/annotation/text.rb +7 -7
  102. data/lib/gcloud/vision/annotation/vertex.rb +1 -1
  103. data/lib/gcloud/vision/image.rb +11 -11
  104. data/lib/gcloud/vision/location.rb +5 -2
  105. data/lib/gcloud/vision/project.rb +14 -16
  106. data/lib/gcloud/vision/service.rb +66 -0
  107. data/lib/google/api_client.rb +0 -0
  108. metadata +27 -24
  109. data/lib/gcloud/bigquery/connection.rb +0 -624
  110. data/lib/gcloud/bigquery/errors.rb +0 -68
  111. data/lib/gcloud/bigquery/table/schema.rb +0 -234
  112. data/lib/gcloud/dns/connection.rb +0 -173
  113. data/lib/gcloud/dns/errors.rb +0 -68
  114. data/lib/gcloud/resource_manager/connection.rb +0 -134
  115. data/lib/gcloud/resource_manager/errors.rb +0 -68
  116. data/lib/gcloud/storage/connection.rb +0 -444
  117. data/lib/gcloud/translate/connection.rb +0 -85
  118. data/lib/gcloud/translate/errors.rb +0 -68
  119. data/lib/gcloud/upload.rb +0 -95
  120. data/lib/gcloud/vision/connection.rb +0 -63
  121. data/lib/gcloud/vision/errors.rb +0 -69
@@ -99,13 +99,16 @@ module Gcloud
99
99
  ##
100
100
  # @private New Google API Client LatLng object.
101
101
  def to_gapi
102
- to_hash
102
+ Google::Apis::VisionV1::LatLng.new(
103
+ latitude: latitude,
104
+ longitude: longitude
105
+ )
103
106
  end
104
107
 
105
108
  ##
106
109
  # @private New Location from a Google API Client LatLng object.
107
110
  def self.from_gapi gapi
108
- new gapi["latitude"], gapi["longitude"]
111
+ new gapi.latitude, gapi.longitude
109
112
  end
110
113
  end
111
114
  end
@@ -14,12 +14,12 @@
14
14
 
15
15
 
16
16
  require "gcloud/gce"
17
- require "gcloud/vision/connection"
17
+ require "gcloud/errors"
18
+ require "gcloud/vision/service"
18
19
  require "gcloud/vision/credentials"
19
20
  require "gcloud/vision/annotate"
20
21
  require "gcloud/vision/image"
21
22
  require "gcloud/vision/annotation"
22
- require "gcloud/vision/errors"
23
23
 
24
24
  module Gcloud
25
25
  module Vision
@@ -48,15 +48,13 @@ module Gcloud
48
48
  # See Gcloud#vision
49
49
  class Project
50
50
  ##
51
- # @private The Connection object.
52
- attr_accessor :connection
51
+ # @private The Service object.
52
+ attr_accessor :service
53
53
 
54
54
  ##
55
55
  # @private Creates a new Project instance.
56
- def initialize project, credentials
57
- project = project.to_s # Always cast to a string
58
- fail ArgumentError, "project is missing" if project.empty?
59
- @connection = Connection.new project, credentials
56
+ def initialize service
57
+ @service = service
60
58
  end
61
59
 
62
60
  # The Vision project connected to.
@@ -71,7 +69,7 @@ module Gcloud
71
69
  # vision.project #=> "my-todo-project"
72
70
  #
73
71
  def project
74
- connection.project
72
+ service.project
75
73
  end
76
74
 
77
75
  ##
@@ -132,7 +130,7 @@ module Gcloud
132
130
  ##
133
131
  # Performs detection of Cloud Vision [features](https://cloud.google.com/vision/reference/rest/v1/images/annotate#Feature)
134
132
  # on the given image(s). If no options for features are provided, **all**
135
- # image detection features will be performed, with a default of `10`
133
+ # image detection features will be performed, with a default of `100`
136
134
  # results for faces, landmarks, logos, and labels. If any feature option
137
135
  # is provided, only the specified feature detections will be performed.
138
136
  # Please review [Pricing](https://cloud.google.com/vision/docs/pricing)
@@ -257,10 +255,10 @@ module Gcloud
257
255
 
258
256
  yield a if block_given?
259
257
 
260
- resp = connection.annotate a.requests
261
- fail ApiError.from_response(resp) unless resp.success?
262
- annotations = Array(resp.data["responses"]).map do |gapi|
263
- Annotation.from_gapi gapi
258
+ gapi = service.annotate a.requests
259
+ annotations = Array(gapi.responses).map do |g|
260
+ fail Error.from_error(g.error) if g.error
261
+ Annotation.from_gapi g
264
262
  end
265
263
  return annotations.first if annotations.count == 1
266
264
  annotations
@@ -272,8 +270,8 @@ module Gcloud
272
270
 
273
271
  ##
274
272
  # Raise an error unless an active connection is available.
275
- def ensure_connection!
276
- fail "Must have active connection" unless connection
273
+ def ensure_service!
274
+ fail "Must have active connection" unless service
277
275
  end
278
276
  end
279
277
  end
@@ -0,0 +1,66 @@
1
+ # Copyright 2016 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ require "gcloud/version"
17
+ require "gcloud/errors"
18
+ require "google/apis/vision_v1"
19
+
20
+ module Gcloud
21
+ module Vision
22
+ ##
23
+ # @private
24
+ # Represents the service to Vision, exposing the API calls.
25
+ class Service
26
+ ##
27
+ # Alias to the Google Client API module
28
+ API = Google::Apis::VisionV1
29
+
30
+ attr_accessor :project
31
+ attr_accessor :credentials
32
+
33
+ ##
34
+ # Creates a new Service instance.
35
+ def initialize project, credentials, retries: nil, timeout: nil
36
+ @project = project
37
+ @credentials = credentials
38
+ @service = API::VisionService.new
39
+ @service.client_options.application_name = "gcloud-ruby"
40
+ @service.client_options.application_version = Gcloud::VERSION
41
+ @service.request_options.retries = retries || 3
42
+ @service.request_options.timeout_sec = timeout if timeout
43
+ @service.authorization = @credentials.client
44
+ end
45
+
46
+ def service
47
+ return mocked_service if mocked_service
48
+ @service
49
+ end
50
+ attr_accessor :mocked_service
51
+
52
+ ##
53
+ # Returns API::BatchAnnotateImagesResponse
54
+ def annotate requests
55
+ request = API::BatchAnnotateImagesRequest.new(requests: requests)
56
+ service.annotate_image request
57
+ rescue Google::Apis::Error => e
58
+ raise Gcloud::Error.from_error(e)
59
+ end
60
+
61
+ def inspect
62
+ "#{self.class}(#{@project})"
63
+ end
64
+ end
65
+ end
66
+ end
File without changes
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gcloud
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.11.0
4
+ version: 0.12.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Silvano Luciani
@@ -10,22 +10,22 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2016-06-14 00:00:00.000000000 Z
13
+ date: 2016-07-25 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: grpc
17
17
  requirement: !ruby/object:Gem::Requirement
18
18
  requirements:
19
- - - ~>
19
+ - - '='
20
20
  - !ruby/object:Gem::Version
21
- version: 0.13.1
21
+ version: 1.0.0.pre1
22
22
  type: :runtime
23
23
  prerelease: false
24
24
  version_requirements: !ruby/object:Gem::Requirement
25
25
  requirements:
26
- - - ~>
26
+ - - '='
27
27
  - !ruby/object:Gem::Version
28
- version: 0.13.1
28
+ version: 1.0.0.pre1
29
29
  - !ruby/object:Gem::Dependency
30
30
  name: google-protobuf
31
31
  requirement: !ruby/object:Gem::Requirement
@@ -46,28 +46,34 @@ dependencies:
46
46
  requirements:
47
47
  - - ~>
48
48
  - !ruby/object:Gem::Version
49
- version: 0.8.3
49
+ version: 0.9.11
50
50
  type: :runtime
51
51
  prerelease: false
52
52
  version_requirements: !ruby/object:Gem::Requirement
53
53
  requirements:
54
54
  - - ~>
55
55
  - !ruby/object:Gem::Version
56
- version: 0.8.3
56
+ version: 0.9.11
57
57
  - !ruby/object:Gem::Dependency
58
58
  name: mime-types
59
59
  requirement: !ruby/object:Gem::Requirement
60
60
  requirements:
61
- - - ~>
61
+ - - ! '>='
62
62
  - !ruby/object:Gem::Version
63
63
  version: '2.4'
64
+ - - <
65
+ - !ruby/object:Gem::Version
66
+ version: '4.0'
64
67
  type: :runtime
65
68
  prerelease: false
66
69
  version_requirements: !ruby/object:Gem::Requirement
67
70
  requirements:
68
- - - ~>
71
+ - - ! '>='
69
72
  - !ruby/object:Gem::Version
70
73
  version: '2.4'
74
+ - - <
75
+ - !ruby/object:Gem::Version
76
+ version: '4.0'
71
77
  - !ruby/object:Gem::Dependency
72
78
  name: digest-crc
73
79
  requirement: !ruby/object:Gem::Requirement
@@ -282,14 +288,12 @@ files:
282
288
  - lib/gcloud.rb
283
289
  - lib/gcloud/backoff.rb
284
290
  - lib/gcloud/bigquery.rb
285
- - lib/gcloud/bigquery/connection.rb
286
291
  - lib/gcloud/bigquery/copy_job.rb
287
292
  - lib/gcloud/bigquery/credentials.rb
288
293
  - lib/gcloud/bigquery/data.rb
289
294
  - lib/gcloud/bigquery/dataset.rb
290
295
  - lib/gcloud/bigquery/dataset/access.rb
291
296
  - lib/gcloud/bigquery/dataset/list.rb
292
- - lib/gcloud/bigquery/errors.rb
293
297
  - lib/gcloud/bigquery/extract_job.rb
294
298
  - lib/gcloud/bigquery/insert_response.rb
295
299
  - lib/gcloud/bigquery/job.rb
@@ -298,9 +302,10 @@ files:
298
302
  - lib/gcloud/bigquery/project.rb
299
303
  - lib/gcloud/bigquery/query_data.rb
300
304
  - lib/gcloud/bigquery/query_job.rb
305
+ - lib/gcloud/bigquery/schema.rb
306
+ - lib/gcloud/bigquery/service.rb
301
307
  - lib/gcloud/bigquery/table.rb
302
308
  - lib/gcloud/bigquery/table/list.rb
303
- - lib/gcloud/bigquery/table/schema.rb
304
309
  - lib/gcloud/bigquery/view.rb
305
310
  - lib/gcloud/credentials.rb
306
311
  - lib/gcloud/datastore.rb
@@ -322,13 +327,12 @@ files:
322
327
  - lib/gcloud/dns.rb
323
328
  - lib/gcloud/dns/change.rb
324
329
  - lib/gcloud/dns/change/list.rb
325
- - lib/gcloud/dns/connection.rb
326
330
  - lib/gcloud/dns/credentials.rb
327
- - lib/gcloud/dns/errors.rb
328
331
  - lib/gcloud/dns/importer.rb
329
332
  - lib/gcloud/dns/project.rb
330
333
  - lib/gcloud/dns/record.rb
331
334
  - lib/gcloud/dns/record/list.rb
335
+ - lib/gcloud/dns/service.rb
332
336
  - lib/gcloud/dns/zone.rb
333
337
  - lib/gcloud/dns/zone/list.rb
334
338
  - lib/gcloud/dns/zone/transaction.rb
@@ -354,28 +358,28 @@ files:
354
358
  - lib/gcloud/pubsub.rb
355
359
  - lib/gcloud/pubsub/credentials.rb
356
360
  - lib/gcloud/pubsub/message.rb
361
+ - lib/gcloud/pubsub/policy.rb
357
362
  - lib/gcloud/pubsub/project.rb
358
363
  - lib/gcloud/pubsub/received_message.rb
359
364
  - lib/gcloud/pubsub/service.rb
360
365
  - lib/gcloud/pubsub/subscription.rb
361
366
  - lib/gcloud/pubsub/subscription/list.rb
362
367
  - lib/gcloud/pubsub/topic.rb
363
- - lib/gcloud/pubsub/topic/batch.rb
364
368
  - lib/gcloud/pubsub/topic/list.rb
369
+ - lib/gcloud/pubsub/topic/publisher.rb
365
370
  - lib/gcloud/resource_manager.rb
366
- - lib/gcloud/resource_manager/connection.rb
367
371
  - lib/gcloud/resource_manager/credentials.rb
368
- - lib/gcloud/resource_manager/errors.rb
369
372
  - lib/gcloud/resource_manager/manager.rb
373
+ - lib/gcloud/resource_manager/policy.rb
370
374
  - lib/gcloud/resource_manager/project.rb
371
375
  - lib/gcloud/resource_manager/project/list.rb
372
376
  - lib/gcloud/resource_manager/project/updater.rb
377
+ - lib/gcloud/resource_manager/service.rb
373
378
  - lib/gcloud/storage.rb
374
379
  - lib/gcloud/storage/bucket.rb
375
380
  - lib/gcloud/storage/bucket/acl.rb
376
381
  - lib/gcloud/storage/bucket/cors.rb
377
382
  - lib/gcloud/storage/bucket/list.rb
378
- - lib/gcloud/storage/connection.rb
379
383
  - lib/gcloud/storage/credentials.rb
380
384
  - lib/gcloud/storage/errors.rb
381
385
  - lib/gcloud/storage/file.rb
@@ -383,14 +387,13 @@ files:
383
387
  - lib/gcloud/storage/file/list.rb
384
388
  - lib/gcloud/storage/file/verifier.rb
385
389
  - lib/gcloud/storage/project.rb
390
+ - lib/gcloud/storage/service.rb
386
391
  - lib/gcloud/translate.rb
387
392
  - lib/gcloud/translate/api.rb
388
- - lib/gcloud/translate/connection.rb
389
393
  - lib/gcloud/translate/detection.rb
390
- - lib/gcloud/translate/errors.rb
391
394
  - lib/gcloud/translate/language.rb
395
+ - lib/gcloud/translate/service.rb
392
396
  - lib/gcloud/translate/translation.rb
393
- - lib/gcloud/upload.rb
394
397
  - lib/gcloud/version.rb
395
398
  - lib/gcloud/vision.rb
396
399
  - lib/gcloud/vision/annotate.rb
@@ -401,16 +404,16 @@ files:
401
404
  - lib/gcloud/vision/annotation/safe_search.rb
402
405
  - lib/gcloud/vision/annotation/text.rb
403
406
  - lib/gcloud/vision/annotation/vertex.rb
404
- - lib/gcloud/vision/connection.rb
405
407
  - lib/gcloud/vision/credentials.rb
406
- - lib/gcloud/vision/errors.rb
407
408
  - lib/gcloud/vision/image.rb
408
409
  - lib/gcloud/vision/location.rb
409
410
  - lib/gcloud/vision/project.rb
411
+ - lib/gcloud/vision/service.rb
410
412
  - lib/google/api/annotations.rb
411
413
  - lib/google/api/http.rb
412
414
  - lib/google/api/label.rb
413
415
  - lib/google/api/monitored_resource.rb
416
+ - lib/google/api_client.rb
414
417
  - lib/google/datastore/v1beta3/datastore.rb
415
418
  - lib/google/datastore/v1beta3/datastore_services.rb
416
419
  - lib/google/datastore/v1beta3/entity.rb
@@ -1,624 +0,0 @@
1
- # Copyright 2015 Google Inc. All rights reserved.
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
-
15
-
16
- require "pathname"
17
- require "gcloud/version"
18
- require "gcloud/backoff"
19
- require "google/api_client"
20
- require "digest/md5"
21
-
22
- module Gcloud
23
- module Bigquery
24
- ##
25
- # @private Represents the connection to Bigquery,
26
- # as well as expose the API calls.
27
- class Connection
28
- API_VERSION = "v2"
29
-
30
- attr_accessor :project
31
- attr_accessor :credentials
32
-
33
- ##
34
- # Creates a new Connection instance.
35
- def initialize project, credentials
36
- @project = project
37
- @credentials = credentials
38
- @client = Google::APIClient.new application_name: "gcloud-ruby",
39
- application_version: Gcloud::VERSION
40
- @client.authorization = @credentials.client
41
- @bigquery = @client.discovered_api "bigquery", API_VERSION
42
- end
43
-
44
- ##
45
- # Lists all datasets in the specified project to which you have
46
- # been granted the READER dataset role.
47
- def list_datasets options = {}
48
- params = { projectId: @project,
49
- all: options.delete(:all),
50
- pageToken: options.delete(:token),
51
- maxResults: options.delete(:max)
52
- }.delete_if { |_, v| v.nil? }
53
-
54
- execute(
55
- api_method: @bigquery.datasets.list,
56
- parameters: params
57
- )
58
- end
59
-
60
- ##
61
- # Returns the dataset specified by datasetID.
62
- def get_dataset dataset_id
63
- execute(
64
- api_method: @bigquery.datasets.get,
65
- parameters: { projectId: @project, datasetId: dataset_id }
66
- )
67
- end
68
-
69
- ##
70
- # Creates a new empty dataset.
71
- def insert_dataset dataset_id, options = {}
72
- execute(
73
- api_method: @bigquery.datasets.insert,
74
- parameters: { projectId: @project },
75
- body_object: insert_dataset_request(dataset_id, options)
76
- )
77
- end
78
-
79
- ##
80
- # Updates information in an existing dataset, only replacing
81
- # fields that are provided in the submitted dataset resource.
82
- def patch_dataset dataset_id, options = {}
83
- project_id = options[:project_id] || @project
84
-
85
- execute(
86
- api_method: @bigquery.datasets.patch,
87
- parameters: { projectId: project_id, datasetId: dataset_id },
88
- body_object: patch_dataset_request(options)
89
- )
90
- end
91
-
92
- ##
93
- # Deletes the dataset specified by the datasetId value.
94
- # Before you can delete a dataset, you must delete all its tables,
95
- # either manually or by specifying force: true in options.
96
- # Immediately after deletion, you can create another dataset with
97
- # the same name.
98
- def delete_dataset dataset_id, force = nil
99
- execute(
100
- api_method: @bigquery.datasets.delete,
101
- parameters: { projectId: @project, datasetId: dataset_id,
102
- deleteContents: force
103
- }.delete_if { |_, v| v.nil? }
104
- )
105
- end
106
-
107
- ##
108
- # Lists all tables in the specified dataset.
109
- # Requires the READER dataset role.
110
- def list_tables dataset_id, options = {}
111
- params = { projectId: @project,
112
- datasetId: dataset_id,
113
- pageToken: options.delete(:token),
114
- maxResults: options.delete(:max)
115
- }.delete_if { |_, v| v.nil? }
116
-
117
- execute(
118
- api_method: @bigquery.tables.list,
119
- parameters: params
120
- )
121
- end
122
-
123
- def get_project_table project_id, dataset_id, table_id
124
- execute(
125
- api_method: @bigquery.tables.get,
126
- parameters: { projectId: project_id, datasetId: dataset_id,
127
- tableId: table_id }
128
- )
129
- end
130
-
131
- ##
132
- # Gets the specified table resource by table ID.
133
- # This method does not return the data in the table,
134
- # it only returns the table resource,
135
- # which describes the structure of this table.
136
- def get_table dataset_id, table_id
137
- get_project_table @project, dataset_id, table_id
138
- end
139
-
140
- ##
141
- # Creates a new, empty table in the dataset.
142
- def insert_table dataset_id, table_id, options = {}
143
- execute(
144
- api_method: @bigquery.tables.insert,
145
- parameters: { projectId: @project, datasetId: dataset_id },
146
- body_object: insert_table_request(dataset_id, table_id, options)
147
- )
148
- end
149
-
150
- ##
151
- # Updates information in an existing table, replacing fields that
152
- # are provided in the submitted table resource.
153
- def patch_table dataset_id, table_id, options = {}
154
- execute(
155
- api_method: @bigquery.tables.patch,
156
- parameters: { projectId: @project, datasetId: dataset_id,
157
- tableId: table_id },
158
- body_object: patch_table_request(options)
159
- )
160
- end
161
-
162
- ##
163
- # Deletes the table specified by tableId from the dataset.
164
- # If the table contains data, all the data will be deleted.
165
- def delete_table dataset_id, table_id
166
- execute(
167
- api_method: @bigquery.tables.delete,
168
- parameters: { projectId: @project, datasetId: dataset_id,
169
- tableId: table_id }
170
- )
171
- end
172
-
173
- ##
174
- # Retrieves data from the table.
175
- def list_tabledata dataset_id, table_id, options = {}
176
- params = { projectId: @project,
177
- datasetId: dataset_id, tableId: table_id,
178
- pageToken: options.delete(:token),
179
- maxResults: options.delete(:max),
180
- startIndex: options.delete(:start)
181
- }.delete_if { |_, v| v.nil? }
182
-
183
- execute(
184
- api_method: @bigquery.tabledata.list,
185
- parameters: params
186
- )
187
- end
188
-
189
- def insert_tabledata dataset_id, table_id, rows, options = {}
190
- execute(
191
- api_method: @bigquery.tabledata.insert_all,
192
- parameters: { projectId: @project,
193
- datasetId: dataset_id,
194
- tableId: table_id },
195
- body_object: insert_tabledata_rows(rows, options)
196
- )
197
- end
198
-
199
- ##
200
- # Lists all jobs in the specified project to which you have
201
- # been granted the READER job role.
202
- def list_jobs options = {}
203
- execute(
204
- api_method: @bigquery.jobs.list,
205
- parameters: list_jobs_params(options)
206
- )
207
- end
208
-
209
- ##
210
- # Returns the job specified by jobID.
211
- def get_job job_id
212
- execute(
213
- api_method: @bigquery.jobs.get,
214
- parameters: { projectId: @project, jobId: job_id }
215
- )
216
- end
217
-
218
- def insert_job config
219
- execute(
220
- api_method: @bigquery.jobs.insert,
221
- parameters: { projectId: @project },
222
- body_object: { "configuration" => config }
223
- )
224
- end
225
-
226
- def query_job query, options = {}
227
- execute(
228
- api_method: @bigquery.jobs.insert,
229
- parameters: { projectId: @project },
230
- body_object: query_table_config(query, options)
231
- )
232
- end
233
-
234
- def query query, options = {}
235
- execute(
236
- api_method: @bigquery.jobs.query,
237
- parameters: { projectId: @project },
238
- body_object: query_config(query, options)
239
- )
240
- end
241
-
242
- ##
243
- # Returns the query data for the job
244
- def job_query_results job_id, options = {}
245
- params = { projectId: @project, jobId: job_id,
246
- pageToken: options.delete(:token),
247
- maxResults: options.delete(:max),
248
- startIndex: options.delete(:start),
249
- timeoutMs: options.delete(:timeout)
250
- }.delete_if { |_, v| v.nil? }
251
-
252
- execute(
253
- api_method: @bigquery.jobs.get_query_results,
254
- parameters: params
255
- )
256
- end
257
-
258
- def copy_table source, target, options = {}
259
- execute(
260
- api_method: @bigquery.jobs.insert,
261
- parameters: { projectId: @project },
262
- body_object: copy_table_config(source, target, options)
263
- )
264
- end
265
-
266
- def link_table table, urls, options = {}
267
- execute(
268
- api_method: @bigquery.jobs.insert,
269
- parameters: { projectId: @project },
270
- body_object: link_table_config(table, urls, options)
271
- )
272
- end
273
-
274
- def extract_table table, storage_files, options = {}
275
- execute(
276
- api_method: @bigquery.jobs.insert,
277
- parameters: { projectId: @project },
278
- body_object: extract_table_config(table, storage_files, options)
279
- )
280
- end
281
-
282
- def load_table table, storage_url, options = {}
283
- execute(
284
- api_method: @bigquery.jobs.insert,
285
- parameters: { projectId: @project },
286
- body_object: load_table_config(table, storage_url,
287
- Array(storage_url).first, options)
288
- )
289
- end
290
-
291
- def load_multipart table, file, options = {}
292
- media = load_media file
293
-
294
- execute(
295
- api_method: @bigquery.jobs.insert,
296
- media: media,
297
- parameters: { projectId: @project, uploadType: "multipart" },
298
- body_object: load_table_config(table, nil, file, options)
299
- )
300
- end
301
-
302
- def load_resumable table, file, chunk_size = nil, options = {}
303
- media = load_media file, chunk_size
304
-
305
- result = execute(
306
- api_method: @bigquery.jobs.insert,
307
- media: media,
308
- parameters: { projectId: @project, uploadType: "resumable" },
309
- body_object: load_table_config(table, nil, file, options)
310
- )
311
- upload = result.resumable_upload
312
- result = execute upload while upload.resumable?
313
- result
314
- end
315
-
316
- def default_access_rules
317
- [
318
- { "role" => "OWNER", "specialGroup" => "projectOwners" },
319
- { "role" => "WRITER", "specialGroup" => "projectWriters" },
320
- { "role" => "READER", "specialGroup" => "projectReaders" },
321
- { "role" => "OWNER", "userByEmail" => credentials.issuer }
322
- ]
323
- end
324
-
325
- ##
326
- # Extracts at least `tbl` group, and possibly `dts` and `prj` groups,
327
- # from strings in the formats: "my_table", "my_dataset.my_table", or
328
- # "my-project:my_dataset.my_table". Then merges project_id and
329
- # dataset_id from the default table if they are missing.
330
- def self.table_ref_from_s str, default_table_ref
331
- str = str.to_s
332
- m = /\A(((?<prj>\S*):)?(?<dts>\S*)\.)?(?<tbl>\S*)\z/.match str
333
- unless m
334
- fail ArgumentError, "unable to identify table from #{str.inspect}"
335
- end
336
- str_table_ref = { "projectId" => m["prj"],
337
- "datasetId" => m["dts"],
338
- "tableId" => m["tbl"] }.delete_if { |_, v| v.nil? }
339
- default_table_ref.merge str_table_ref
340
- end
341
-
342
- def inspect
343
- "#{self.class}(#{@project})"
344
- end
345
-
346
- protected
347
-
348
- ##
349
- # Create the HTTP body for insert dataset
350
- def insert_dataset_request dataset_id, options = {}
351
- {
352
- "kind" => "bigquery#dataset",
353
- "datasetReference" => {
354
- "projectId" => @project,
355
- "datasetId" => dataset_id },
356
- "friendlyName" => options[:name],
357
- "description" => options[:description],
358
- "defaultTableExpirationMs" => options[:expiration],
359
- "access" => options[:access],
360
- "location" => options[:location]
361
- }.delete_if { |_, v| v.nil? }
362
- end
363
-
364
- def patch_dataset_request options = {}
365
- {
366
- friendlyName: options[:name],
367
- description: options[:description],
368
- defaultTableExpirationMs: options[:default_expiration],
369
- access: options[:access]
370
- }.delete_if { |_, v| v.nil? }
371
- end
372
-
373
- ##
374
- # The parameters for the list_jobs call.
375
- def list_jobs_params options = {}
376
- params = { projectId: @project,
377
- allUsers: options.delete(:all),
378
- pageToken: options.delete(:token),
379
- maxResults: options.delete(:max),
380
- stateFilter: options.delete(:filter),
381
- projection: "full"
382
- }.delete_if { |_, v| v.nil? }
383
- params
384
- end
385
-
386
- ##
387
- # Create the HTTP body for insert table
388
- def insert_table_request dataset_id, table_id, options = {}
389
- hash = {
390
- tableReference: {
391
- projectId: @project, datasetId: dataset_id, tableId: table_id
392
- },
393
- friendlyName: options[:name],
394
- description: options[:description],
395
- schema: options[:schema]
396
- }.delete_if { |_, v| v.nil? }
397
- hash["view"] = { "query" => options[:query] } if options[:query]
398
- hash
399
- end
400
-
401
- def patch_table_request options = {}
402
- body = { friendlyName: options[:name],
403
- description: options[:description],
404
- schema: options[:schema]
405
- }.delete_if { |_, v| v.nil? }
406
- body["view"] = { "query" => options[:query] } if options[:query]
407
- body
408
- end
409
-
410
- def insert_tabledata_rows rows, options = {}
411
- {
412
- "kind" => "bigquery#tableDataInsertAllRequest",
413
- "skipInvalidRows" => options[:skip_invalid],
414
- "ignoreUnknownValues" => options[:ignore_unknown],
415
- "rows" => rows.map do |row|
416
- { "insertId" => Digest::MD5.base64digest(row.inspect),
417
- "json" => row }
418
- end
419
- }.delete_if { |_, v| v.nil? }
420
- end
421
-
422
- # rubocop:disable all
423
- # Disabled rubocop because the API is verbose and so these methods
424
- # are going to be verbose.
425
-
426
- ##
427
- # Job description for query job
428
- def query_table_config query, options
429
- dest_table = nil
430
- if options[:table]
431
- dest_table = { "projectId" => options[:table].project_id,
432
- "datasetId" => options[:table].dataset_id,
433
- "tableId" => options[:table].table_id }
434
- end
435
- default_dataset = nil
436
- if dataset = options[:dataset]
437
- if dataset.respond_to? :dataset_id
438
- default_dataset = { "projectId" => dataset.project_id,
439
- "datasetId" => dataset.dataset_id }
440
- else
441
- default_dataset = { "datasetId" => dataset }
442
- end
443
- end
444
- {
445
- "configuration" => {
446
- "query" => {
447
- "query" => query,
448
- # "tableDefinitions" => { ... },
449
- "priority" => priority_value(options[:priority]),
450
- "useQueryCache" => options[:cache],
451
- "destinationTable" => dest_table,
452
- "createDisposition" => create_disposition(options[:create]),
453
- "writeDisposition" => write_disposition(options[:write]),
454
- "allowLargeResults" => options[:large_results],
455
- "flattenResults" => options[:flatten],
456
- "defaultDataset" => default_dataset
457
- }.delete_if { |_, v| v.nil? }
458
- }.delete_if { |_, v| v.nil? }
459
- }
460
- end
461
-
462
- def query_config query, options = {}
463
- dataset_config = nil
464
- dataset_config = {
465
- "datasetId" => options[:dataset],
466
- "projectId" => options[:project] || @project
467
- } if options[:dataset]
468
-
469
- {
470
- "kind" => "bigquery#queryRequest",
471
- "query" => query,
472
- "maxResults" => options[:max],
473
- "defaultDataset" => dataset_config,
474
- "timeoutMs" => options[:timeout],
475
- "dryRun" => options[:dryrun],
476
- "useQueryCache" => options[:cache]
477
- }.delete_if { |_, v| v.nil? }
478
- end
479
-
480
- ##
481
- # Job description for copy job
482
- def copy_table_config source, target, options = {}
483
- {
484
- "configuration" => {
485
- "copy" => {
486
- "sourceTable" => source,
487
- "destinationTable" => target,
488
- "createDisposition" => create_disposition(options[:create]),
489
- "writeDisposition" => write_disposition(options[:write])
490
- }.delete_if { |_, v| v.nil? },
491
- "dryRun" => options[:dryrun]
492
- }.delete_if { |_, v| v.nil? }
493
- }
494
- end
495
-
496
- def link_table_config table, urls, options = {}
497
- path = Array(urls).first
498
- {
499
- "configuration" => {
500
- "link" => {
501
- "sourceUri" => Array(urls),
502
- "destinationTable" => table,
503
- "createDisposition" => create_disposition(options[:create]),
504
- "writeDisposition" => write_disposition(options[:write]),
505
- "sourceFormat" => source_format(path, options[:format])
506
- }.delete_if { |_, v| v.nil? },
507
- "dryRun" => options[:dryrun]
508
- }.delete_if { |_, v| v.nil? }
509
- }
510
- end
511
-
512
- def extract_table_config table, storage_files, options = {}
513
- storage_urls = Array(storage_files).map do |url|
514
- url.respond_to?(:to_gs_url) ? url.to_gs_url : url
515
- end
516
- dest_format = source_format storage_urls.first, options[:format]
517
- {
518
- "configuration" => {
519
- "extract" => {
520
- "destinationUris" => Array(storage_urls),
521
- "sourceTable" => table,
522
- "destinationFormat" => dest_format,
523
- "compression" => options[:compression],
524
- "fieldDelimiter" => options[:delimiter],
525
- "printHeader" => options[:header]
526
- }.delete_if { |_, v| v.nil? },
527
- "dryRun" => options[:dryrun]
528
- }.delete_if { |_, v| v.nil? }
529
- }
530
- end
531
-
532
- def load_table_config table, urls, file, options = {}
533
- path = Array(urls).first
534
- path = Pathname(file).to_path unless file.nil?
535
- {
536
- "configuration" => {
537
- "load" => {
538
- "sourceUris" => Array(urls),
539
- "destinationTable" => table,
540
- "createDisposition" => create_disposition(options[:create]),
541
- "writeDisposition" => write_disposition(options[:write]),
542
- "sourceFormat" => source_format(path, options[:format]),
543
- "projectionFields" => projection_fields(options[:projection_fields]),
544
- "allowJaggedRows" => options[:jagged_rows],
545
- "allowQuotedNewlines" => options[:quoted_newlines],
546
- "encoding" => options[:encoding],
547
- "fieldDelimiter" => options[:delimiter],
548
- "ignoreUnknownValues" => options[:ignore_unknown],
549
- "maxBadRecords" => options[:max_bad_records],
550
- "quote" => options[:quote],
551
- "schema" => options[:schema],
552
- "skipLeadingRows" => options[:skip_leading]
553
- }.delete_if { |_, v| v.nil? },
554
- "dryRun" => options[:dryrun]
555
- }.delete_if { |_, v| v.nil? }
556
- }
557
- end
558
-
559
- def create_disposition str
560
- { "create_if_needed" => "CREATE_IF_NEEDED",
561
- "createifneeded" => "CREATE_IF_NEEDED",
562
- "if_needed" => "CREATE_IF_NEEDED",
563
- "needed" => "CREATE_IF_NEEDED",
564
- "create_never" => "CREATE_NEVER",
565
- "createnever" => "CREATE_NEVER",
566
- "never" => "CREATE_NEVER" }[str.to_s.downcase]
567
- end
568
-
569
- def write_disposition str
570
- { "write_truncate" => "WRITE_TRUNCATE",
571
- "writetruncate" => "WRITE_TRUNCATE",
572
- "truncate" => "WRITE_TRUNCATE",
573
- "write_append" => "WRITE_APPEND",
574
- "writeappend" => "WRITE_APPEND",
575
- "append" => "WRITE_APPEND",
576
- "write_empty" => "WRITE_EMPTY",
577
- "writeempty" => "WRITE_EMPTY",
578
- "empty" => "WRITE_EMPTY" }[str.to_s.downcase]
579
- end
580
-
581
- def priority_value str
582
- { "batch" => "BATCH",
583
- "interactive" => "INTERACTIVE" }[str.to_s.downcase]
584
- end
585
-
586
- def source_format path, format
587
- val = { "csv" => "CSV",
588
- "json" => "NEWLINE_DELIMITED_JSON",
589
- "newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
590
- "avro" => "AVRO",
591
- "datastore" => "DATASTORE_BACKUP",
592
- "datastore_backup" => "DATASTORE_BACKUP"}[format.to_s.downcase]
593
- return val unless val.nil?
594
- return nil if path.nil?
595
- return "CSV" if path.end_with? ".csv"
596
- return "NEWLINE_DELIMITED_JSON" if path.end_with? ".json"
597
- return "AVRO" if path.end_with? ".avro"
598
- return "DATASTORE_BACKUP" if path.end_with? ".backup_info"
599
- nil
600
- end
601
-
602
- def projection_fields array_or_str
603
- Array(array_or_str) unless array_or_str.nil?
604
- end
605
-
606
- # rubocop:enable all
607
-
608
- def load_media file, chunk_size = nil
609
- local_path = Pathname(file).to_path
610
- mime_type = "application/octet-stream"
611
-
612
- media = Google::APIClient::UploadIO.new local_path, mime_type
613
- media.chunk_size = chunk_size unless chunk_size.nil?
614
- media
615
- end
616
-
617
- def execute options
618
- Gcloud::Backoff.new.execute_gapi do
619
- @client.execute options
620
- end
621
- end
622
- end
623
- end
624
- end