gcloud 0.11.0 → 0.12.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (121) hide show
  1. checksums.yaml +8 -8
  2. data/AUTHENTICATION.md +3 -3
  3. data/CHANGELOG.md +92 -0
  4. data/OVERVIEW.md +3 -3
  5. data/lib/gcloud.rb +75 -25
  6. data/lib/gcloud/backoff.rb +5 -1
  7. data/lib/gcloud/bigquery.rb +25 -43
  8. data/lib/gcloud/bigquery/copy_job.rb +13 -13
  9. data/lib/gcloud/bigquery/data.rb +20 -16
  10. data/lib/gcloud/bigquery/dataset.rb +202 -177
  11. data/lib/gcloud/bigquery/dataset/access.rb +118 -104
  12. data/lib/gcloud/bigquery/dataset/list.rb +14 -18
  13. data/lib/gcloud/bigquery/extract_job.rb +12 -12
  14. data/lib/gcloud/bigquery/insert_response.rb +12 -14
  15. data/lib/gcloud/bigquery/job.rb +45 -57
  16. data/lib/gcloud/bigquery/job/list.rb +18 -24
  17. data/lib/gcloud/bigquery/load_job.rb +35 -27
  18. data/lib/gcloud/bigquery/project.rb +53 -73
  19. data/lib/gcloud/bigquery/query_data.rb +28 -35
  20. data/lib/gcloud/bigquery/query_job.rb +18 -18
  21. data/lib/gcloud/bigquery/schema.rb +359 -0
  22. data/lib/gcloud/bigquery/service.rb +506 -0
  23. data/lib/gcloud/bigquery/table.rb +185 -266
  24. data/lib/gcloud/bigquery/table/list.rb +15 -19
  25. data/lib/gcloud/bigquery/view.rb +126 -81
  26. data/lib/gcloud/datastore.rb +39 -27
  27. data/lib/gcloud/datastore/commit.rb +2 -2
  28. data/lib/gcloud/datastore/dataset.rb +8 -19
  29. data/lib/gcloud/datastore/dataset/lookup_results.rb +2 -4
  30. data/lib/gcloud/datastore/dataset/query_results.rb +0 -2
  31. data/lib/gcloud/datastore/entity.rb +7 -1
  32. data/lib/gcloud/datastore/errors.rb +5 -27
  33. data/lib/gcloud/datastore/grpc_utils.rb +4 -3
  34. data/lib/gcloud/datastore/key.rb +6 -0
  35. data/lib/gcloud/datastore/service.rb +18 -12
  36. data/lib/gcloud/datastore/transaction.rb +0 -10
  37. data/lib/gcloud/dns.rb +29 -19
  38. data/lib/gcloud/dns/change.rb +10 -15
  39. data/lib/gcloud/dns/change/list.rb +4 -4
  40. data/lib/gcloud/dns/importer.rb +1 -1
  41. data/lib/gcloud/dns/project.rb +32 -49
  42. data/lib/gcloud/dns/record.rb +8 -2
  43. data/lib/gcloud/dns/record/list.rb +4 -4
  44. data/lib/gcloud/dns/service.rb +167 -0
  45. data/lib/gcloud/dns/zone.rb +33 -52
  46. data/lib/gcloud/dns/zone/list.rb +12 -16
  47. data/lib/gcloud/errors.rb +31 -19
  48. data/lib/gcloud/logging.rb +50 -39
  49. data/lib/gcloud/logging/entry.rb +197 -24
  50. data/lib/gcloud/logging/entry/list.rb +0 -2
  51. data/lib/gcloud/logging/logger.rb +1 -1
  52. data/lib/gcloud/logging/metric.rb +3 -9
  53. data/lib/gcloud/logging/metric/list.rb +0 -2
  54. data/lib/gcloud/logging/project.rb +58 -54
  55. data/lib/gcloud/logging/resource_descriptor.rb +2 -2
  56. data/lib/gcloud/logging/resource_descriptor/list.rb +0 -2
  57. data/lib/gcloud/logging/service.rb +32 -23
  58. data/lib/gcloud/logging/sink.rb +8 -14
  59. data/lib/gcloud/logging/sink/list.rb +0 -2
  60. data/lib/gcloud/pubsub.rb +21 -16
  61. data/lib/gcloud/pubsub/policy.rb +204 -0
  62. data/lib/gcloud/pubsub/project.rb +26 -38
  63. data/lib/gcloud/pubsub/service.rb +39 -31
  64. data/lib/gcloud/pubsub/subscription.rb +56 -59
  65. data/lib/gcloud/pubsub/subscription/list.rb +4 -4
  66. data/lib/gcloud/pubsub/topic.rb +69 -66
  67. data/lib/gcloud/pubsub/topic/list.rb +0 -2
  68. data/lib/gcloud/pubsub/topic/{batch.rb → publisher.rb} +15 -2
  69. data/lib/gcloud/resource_manager.rb +27 -26
  70. data/lib/gcloud/resource_manager/manager.rb +19 -39
  71. data/lib/gcloud/resource_manager/policy.rb +211 -0
  72. data/lib/gcloud/resource_manager/project.rb +97 -121
  73. data/lib/gcloud/resource_manager/project/list.rb +7 -7
  74. data/lib/gcloud/resource_manager/project/updater.rb +4 -9
  75. data/lib/gcloud/resource_manager/service.rb +127 -0
  76. data/lib/gcloud/storage.rb +24 -42
  77. data/lib/gcloud/storage/bucket.rb +104 -192
  78. data/lib/gcloud/storage/bucket/acl.rb +47 -143
  79. data/lib/gcloud/storage/bucket/cors.rb +55 -11
  80. data/lib/gcloud/storage/bucket/list.rb +14 -14
  81. data/lib/gcloud/storage/errors.rb +3 -43
  82. data/lib/gcloud/storage/file.rb +114 -111
  83. data/lib/gcloud/storage/file/acl.rb +27 -113
  84. data/lib/gcloud/storage/file/list.rb +21 -21
  85. data/lib/gcloud/storage/project.rb +49 -59
  86. data/lib/gcloud/storage/service.rb +347 -0
  87. data/lib/gcloud/translate.rb +24 -14
  88. data/lib/gcloud/translate/api.rb +12 -21
  89. data/lib/gcloud/translate/detection.rb +5 -5
  90. data/lib/gcloud/translate/language.rb +1 -1
  91. data/lib/gcloud/translate/service.rb +80 -0
  92. data/lib/gcloud/translate/translation.rb +6 -6
  93. data/lib/gcloud/version.rb +1 -1
  94. data/lib/gcloud/vision.rb +24 -15
  95. data/lib/gcloud/vision/annotate.rb +24 -21
  96. data/lib/gcloud/vision/annotation.rb +9 -9
  97. data/lib/gcloud/vision/annotation/entity.rb +11 -11
  98. data/lib/gcloud/vision/annotation/face.rb +25 -25
  99. data/lib/gcloud/vision/annotation/properties.rb +8 -8
  100. data/lib/gcloud/vision/annotation/safe_search.rb +4 -4
  101. data/lib/gcloud/vision/annotation/text.rb +7 -7
  102. data/lib/gcloud/vision/annotation/vertex.rb +1 -1
  103. data/lib/gcloud/vision/image.rb +11 -11
  104. data/lib/gcloud/vision/location.rb +5 -2
  105. data/lib/gcloud/vision/project.rb +14 -16
  106. data/lib/gcloud/vision/service.rb +66 -0
  107. data/lib/google/api_client.rb +0 -0
  108. metadata +27 -24
  109. data/lib/gcloud/bigquery/connection.rb +0 -624
  110. data/lib/gcloud/bigquery/errors.rb +0 -68
  111. data/lib/gcloud/bigquery/table/schema.rb +0 -234
  112. data/lib/gcloud/dns/connection.rb +0 -173
  113. data/lib/gcloud/dns/errors.rb +0 -68
  114. data/lib/gcloud/resource_manager/connection.rb +0 -134
  115. data/lib/gcloud/resource_manager/errors.rb +0 -68
  116. data/lib/gcloud/storage/connection.rb +0 -444
  117. data/lib/gcloud/translate/connection.rb +0 -85
  118. data/lib/gcloud/translate/errors.rb +0 -68
  119. data/lib/gcloud/upload.rb +0 -95
  120. data/lib/gcloud/vision/connection.rb +0 -63
  121. data/lib/gcloud/vision/errors.rb +0 -69
@@ -99,13 +99,16 @@ module Gcloud
99
99
  ##
100
100
  # @private New Google API Client LatLng object.
101
101
  def to_gapi
102
- to_hash
102
+ Google::Apis::VisionV1::LatLng.new(
103
+ latitude: latitude,
104
+ longitude: longitude
105
+ )
103
106
  end
104
107
 
105
108
  ##
106
109
  # @private New Location from a Google API Client LatLng object.
107
110
  def self.from_gapi gapi
108
- new gapi["latitude"], gapi["longitude"]
111
+ new gapi.latitude, gapi.longitude
109
112
  end
110
113
  end
111
114
  end
@@ -14,12 +14,12 @@
14
14
 
15
15
 
16
16
  require "gcloud/gce"
17
- require "gcloud/vision/connection"
17
+ require "gcloud/errors"
18
+ require "gcloud/vision/service"
18
19
  require "gcloud/vision/credentials"
19
20
  require "gcloud/vision/annotate"
20
21
  require "gcloud/vision/image"
21
22
  require "gcloud/vision/annotation"
22
- require "gcloud/vision/errors"
23
23
 
24
24
  module Gcloud
25
25
  module Vision
@@ -48,15 +48,13 @@ module Gcloud
48
48
  # See Gcloud#vision
49
49
  class Project
50
50
  ##
51
- # @private The Connection object.
52
- attr_accessor :connection
51
+ # @private The Service object.
52
+ attr_accessor :service
53
53
 
54
54
  ##
55
55
  # @private Creates a new Project instance.
56
- def initialize project, credentials
57
- project = project.to_s # Always cast to a string
58
- fail ArgumentError, "project is missing" if project.empty?
59
- @connection = Connection.new project, credentials
56
+ def initialize service
57
+ @service = service
60
58
  end
61
59
 
62
60
  # The Vision project connected to.
@@ -71,7 +69,7 @@ module Gcloud
71
69
  # vision.project #=> "my-todo-project"
72
70
  #
73
71
  def project
74
- connection.project
72
+ service.project
75
73
  end
76
74
 
77
75
  ##
@@ -132,7 +130,7 @@ module Gcloud
132
130
  ##
133
131
  # Performs detection of Cloud Vision [features](https://cloud.google.com/vision/reference/rest/v1/images/annotate#Feature)
134
132
  # on the given image(s). If no options for features are provided, **all**
135
- # image detection features will be performed, with a default of `10`
133
+ # image detection features will be performed, with a default of `100`
136
134
  # results for faces, landmarks, logos, and labels. If any feature option
137
135
  # is provided, only the specified feature detections will be performed.
138
136
  # Please review [Pricing](https://cloud.google.com/vision/docs/pricing)
@@ -257,10 +255,10 @@ module Gcloud
257
255
 
258
256
  yield a if block_given?
259
257
 
260
- resp = connection.annotate a.requests
261
- fail ApiError.from_response(resp) unless resp.success?
262
- annotations = Array(resp.data["responses"]).map do |gapi|
263
- Annotation.from_gapi gapi
258
+ gapi = service.annotate a.requests
259
+ annotations = Array(gapi.responses).map do |g|
260
+ fail Error.from_error(g.error) if g.error
261
+ Annotation.from_gapi g
264
262
  end
265
263
  return annotations.first if annotations.count == 1
266
264
  annotations
@@ -272,8 +270,8 @@ module Gcloud
272
270
 
273
271
  ##
274
272
  # Raise an error unless an active connection is available.
275
- def ensure_connection!
276
- fail "Must have active connection" unless connection
273
+ def ensure_service!
274
+ fail "Must have active connection" unless service
277
275
  end
278
276
  end
279
277
  end
@@ -0,0 +1,66 @@
1
+ # Copyright 2016 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ require "gcloud/version"
17
+ require "gcloud/errors"
18
+ require "google/apis/vision_v1"
19
+
20
+ module Gcloud
21
+ module Vision
22
+ ##
23
+ # @private
24
+ # Represents the service to Vision, exposing the API calls.
25
+ class Service
26
+ ##
27
+ # Alias to the Google Client API module
28
+ API = Google::Apis::VisionV1
29
+
30
+ attr_accessor :project
31
+ attr_accessor :credentials
32
+
33
+ ##
34
+ # Creates a new Service instance.
35
+ def initialize project, credentials, retries: nil, timeout: nil
36
+ @project = project
37
+ @credentials = credentials
38
+ @service = API::VisionService.new
39
+ @service.client_options.application_name = "gcloud-ruby"
40
+ @service.client_options.application_version = Gcloud::VERSION
41
+ @service.request_options.retries = retries || 3
42
+ @service.request_options.timeout_sec = timeout if timeout
43
+ @service.authorization = @credentials.client
44
+ end
45
+
46
+ def service
47
+ return mocked_service if mocked_service
48
+ @service
49
+ end
50
+ attr_accessor :mocked_service
51
+
52
+ ##
53
+ # Returns API::BatchAnnotateImagesResponse
54
+ def annotate requests
55
+ request = API::BatchAnnotateImagesRequest.new(requests: requests)
56
+ service.annotate_image request
57
+ rescue Google::Apis::Error => e
58
+ raise Gcloud::Error.from_error(e)
59
+ end
60
+
61
+ def inspect
62
+ "#{self.class}(#{@project})"
63
+ end
64
+ end
65
+ end
66
+ end
File without changes
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gcloud
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.11.0
4
+ version: 0.12.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Silvano Luciani
@@ -10,22 +10,22 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2016-06-14 00:00:00.000000000 Z
13
+ date: 2016-07-25 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: grpc
17
17
  requirement: !ruby/object:Gem::Requirement
18
18
  requirements:
19
- - - ~>
19
+ - - '='
20
20
  - !ruby/object:Gem::Version
21
- version: 0.13.1
21
+ version: 1.0.0.pre1
22
22
  type: :runtime
23
23
  prerelease: false
24
24
  version_requirements: !ruby/object:Gem::Requirement
25
25
  requirements:
26
- - - ~>
26
+ - - '='
27
27
  - !ruby/object:Gem::Version
28
- version: 0.13.1
28
+ version: 1.0.0.pre1
29
29
  - !ruby/object:Gem::Dependency
30
30
  name: google-protobuf
31
31
  requirement: !ruby/object:Gem::Requirement
@@ -46,28 +46,34 @@ dependencies:
46
46
  requirements:
47
47
  - - ~>
48
48
  - !ruby/object:Gem::Version
49
- version: 0.8.3
49
+ version: 0.9.11
50
50
  type: :runtime
51
51
  prerelease: false
52
52
  version_requirements: !ruby/object:Gem::Requirement
53
53
  requirements:
54
54
  - - ~>
55
55
  - !ruby/object:Gem::Version
56
- version: 0.8.3
56
+ version: 0.9.11
57
57
  - !ruby/object:Gem::Dependency
58
58
  name: mime-types
59
59
  requirement: !ruby/object:Gem::Requirement
60
60
  requirements:
61
- - - ~>
61
+ - - ! '>='
62
62
  - !ruby/object:Gem::Version
63
63
  version: '2.4'
64
+ - - <
65
+ - !ruby/object:Gem::Version
66
+ version: '4.0'
64
67
  type: :runtime
65
68
  prerelease: false
66
69
  version_requirements: !ruby/object:Gem::Requirement
67
70
  requirements:
68
- - - ~>
71
+ - - ! '>='
69
72
  - !ruby/object:Gem::Version
70
73
  version: '2.4'
74
+ - - <
75
+ - !ruby/object:Gem::Version
76
+ version: '4.0'
71
77
  - !ruby/object:Gem::Dependency
72
78
  name: digest-crc
73
79
  requirement: !ruby/object:Gem::Requirement
@@ -282,14 +288,12 @@ files:
282
288
  - lib/gcloud.rb
283
289
  - lib/gcloud/backoff.rb
284
290
  - lib/gcloud/bigquery.rb
285
- - lib/gcloud/bigquery/connection.rb
286
291
  - lib/gcloud/bigquery/copy_job.rb
287
292
  - lib/gcloud/bigquery/credentials.rb
288
293
  - lib/gcloud/bigquery/data.rb
289
294
  - lib/gcloud/bigquery/dataset.rb
290
295
  - lib/gcloud/bigquery/dataset/access.rb
291
296
  - lib/gcloud/bigquery/dataset/list.rb
292
- - lib/gcloud/bigquery/errors.rb
293
297
  - lib/gcloud/bigquery/extract_job.rb
294
298
  - lib/gcloud/bigquery/insert_response.rb
295
299
  - lib/gcloud/bigquery/job.rb
@@ -298,9 +302,10 @@ files:
298
302
  - lib/gcloud/bigquery/project.rb
299
303
  - lib/gcloud/bigquery/query_data.rb
300
304
  - lib/gcloud/bigquery/query_job.rb
305
+ - lib/gcloud/bigquery/schema.rb
306
+ - lib/gcloud/bigquery/service.rb
301
307
  - lib/gcloud/bigquery/table.rb
302
308
  - lib/gcloud/bigquery/table/list.rb
303
- - lib/gcloud/bigquery/table/schema.rb
304
309
  - lib/gcloud/bigquery/view.rb
305
310
  - lib/gcloud/credentials.rb
306
311
  - lib/gcloud/datastore.rb
@@ -322,13 +327,12 @@ files:
322
327
  - lib/gcloud/dns.rb
323
328
  - lib/gcloud/dns/change.rb
324
329
  - lib/gcloud/dns/change/list.rb
325
- - lib/gcloud/dns/connection.rb
326
330
  - lib/gcloud/dns/credentials.rb
327
- - lib/gcloud/dns/errors.rb
328
331
  - lib/gcloud/dns/importer.rb
329
332
  - lib/gcloud/dns/project.rb
330
333
  - lib/gcloud/dns/record.rb
331
334
  - lib/gcloud/dns/record/list.rb
335
+ - lib/gcloud/dns/service.rb
332
336
  - lib/gcloud/dns/zone.rb
333
337
  - lib/gcloud/dns/zone/list.rb
334
338
  - lib/gcloud/dns/zone/transaction.rb
@@ -354,28 +358,28 @@ files:
354
358
  - lib/gcloud/pubsub.rb
355
359
  - lib/gcloud/pubsub/credentials.rb
356
360
  - lib/gcloud/pubsub/message.rb
361
+ - lib/gcloud/pubsub/policy.rb
357
362
  - lib/gcloud/pubsub/project.rb
358
363
  - lib/gcloud/pubsub/received_message.rb
359
364
  - lib/gcloud/pubsub/service.rb
360
365
  - lib/gcloud/pubsub/subscription.rb
361
366
  - lib/gcloud/pubsub/subscription/list.rb
362
367
  - lib/gcloud/pubsub/topic.rb
363
- - lib/gcloud/pubsub/topic/batch.rb
364
368
  - lib/gcloud/pubsub/topic/list.rb
369
+ - lib/gcloud/pubsub/topic/publisher.rb
365
370
  - lib/gcloud/resource_manager.rb
366
- - lib/gcloud/resource_manager/connection.rb
367
371
  - lib/gcloud/resource_manager/credentials.rb
368
- - lib/gcloud/resource_manager/errors.rb
369
372
  - lib/gcloud/resource_manager/manager.rb
373
+ - lib/gcloud/resource_manager/policy.rb
370
374
  - lib/gcloud/resource_manager/project.rb
371
375
  - lib/gcloud/resource_manager/project/list.rb
372
376
  - lib/gcloud/resource_manager/project/updater.rb
377
+ - lib/gcloud/resource_manager/service.rb
373
378
  - lib/gcloud/storage.rb
374
379
  - lib/gcloud/storage/bucket.rb
375
380
  - lib/gcloud/storage/bucket/acl.rb
376
381
  - lib/gcloud/storage/bucket/cors.rb
377
382
  - lib/gcloud/storage/bucket/list.rb
378
- - lib/gcloud/storage/connection.rb
379
383
  - lib/gcloud/storage/credentials.rb
380
384
  - lib/gcloud/storage/errors.rb
381
385
  - lib/gcloud/storage/file.rb
@@ -383,14 +387,13 @@ files:
383
387
  - lib/gcloud/storage/file/list.rb
384
388
  - lib/gcloud/storage/file/verifier.rb
385
389
  - lib/gcloud/storage/project.rb
390
+ - lib/gcloud/storage/service.rb
386
391
  - lib/gcloud/translate.rb
387
392
  - lib/gcloud/translate/api.rb
388
- - lib/gcloud/translate/connection.rb
389
393
  - lib/gcloud/translate/detection.rb
390
- - lib/gcloud/translate/errors.rb
391
394
  - lib/gcloud/translate/language.rb
395
+ - lib/gcloud/translate/service.rb
392
396
  - lib/gcloud/translate/translation.rb
393
- - lib/gcloud/upload.rb
394
397
  - lib/gcloud/version.rb
395
398
  - lib/gcloud/vision.rb
396
399
  - lib/gcloud/vision/annotate.rb
@@ -401,16 +404,16 @@ files:
401
404
  - lib/gcloud/vision/annotation/safe_search.rb
402
405
  - lib/gcloud/vision/annotation/text.rb
403
406
  - lib/gcloud/vision/annotation/vertex.rb
404
- - lib/gcloud/vision/connection.rb
405
407
  - lib/gcloud/vision/credentials.rb
406
- - lib/gcloud/vision/errors.rb
407
408
  - lib/gcloud/vision/image.rb
408
409
  - lib/gcloud/vision/location.rb
409
410
  - lib/gcloud/vision/project.rb
411
+ - lib/gcloud/vision/service.rb
410
412
  - lib/google/api/annotations.rb
411
413
  - lib/google/api/http.rb
412
414
  - lib/google/api/label.rb
413
415
  - lib/google/api/monitored_resource.rb
416
+ - lib/google/api_client.rb
414
417
  - lib/google/datastore/v1beta3/datastore.rb
415
418
  - lib/google/datastore/v1beta3/datastore_services.rb
416
419
  - lib/google/datastore/v1beta3/entity.rb
@@ -1,624 +0,0 @@
1
- # Copyright 2015 Google Inc. All rights reserved.
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
-
15
-
16
- require "pathname"
17
- require "gcloud/version"
18
- require "gcloud/backoff"
19
- require "google/api_client"
20
- require "digest/md5"
21
-
22
- module Gcloud
23
- module Bigquery
24
- ##
25
- # @private Represents the connection to Bigquery,
26
- # as well as expose the API calls.
27
- class Connection
28
- API_VERSION = "v2"
29
-
30
- attr_accessor :project
31
- attr_accessor :credentials
32
-
33
- ##
34
- # Creates a new Connection instance.
35
- def initialize project, credentials
36
- @project = project
37
- @credentials = credentials
38
- @client = Google::APIClient.new application_name: "gcloud-ruby",
39
- application_version: Gcloud::VERSION
40
- @client.authorization = @credentials.client
41
- @bigquery = @client.discovered_api "bigquery", API_VERSION
42
- end
43
-
44
- ##
45
- # Lists all datasets in the specified project to which you have
46
- # been granted the READER dataset role.
47
- def list_datasets options = {}
48
- params = { projectId: @project,
49
- all: options.delete(:all),
50
- pageToken: options.delete(:token),
51
- maxResults: options.delete(:max)
52
- }.delete_if { |_, v| v.nil? }
53
-
54
- execute(
55
- api_method: @bigquery.datasets.list,
56
- parameters: params
57
- )
58
- end
59
-
60
- ##
61
- # Returns the dataset specified by datasetID.
62
- def get_dataset dataset_id
63
- execute(
64
- api_method: @bigquery.datasets.get,
65
- parameters: { projectId: @project, datasetId: dataset_id }
66
- )
67
- end
68
-
69
- ##
70
- # Creates a new empty dataset.
71
- def insert_dataset dataset_id, options = {}
72
- execute(
73
- api_method: @bigquery.datasets.insert,
74
- parameters: { projectId: @project },
75
- body_object: insert_dataset_request(dataset_id, options)
76
- )
77
- end
78
-
79
- ##
80
- # Updates information in an existing dataset, only replacing
81
- # fields that are provided in the submitted dataset resource.
82
- def patch_dataset dataset_id, options = {}
83
- project_id = options[:project_id] || @project
84
-
85
- execute(
86
- api_method: @bigquery.datasets.patch,
87
- parameters: { projectId: project_id, datasetId: dataset_id },
88
- body_object: patch_dataset_request(options)
89
- )
90
- end
91
-
92
- ##
93
- # Deletes the dataset specified by the datasetId value.
94
- # Before you can delete a dataset, you must delete all its tables,
95
- # either manually or by specifying force: true in options.
96
- # Immediately after deletion, you can create another dataset with
97
- # the same name.
98
- def delete_dataset dataset_id, force = nil
99
- execute(
100
- api_method: @bigquery.datasets.delete,
101
- parameters: { projectId: @project, datasetId: dataset_id,
102
- deleteContents: force
103
- }.delete_if { |_, v| v.nil? }
104
- )
105
- end
106
-
107
- ##
108
- # Lists all tables in the specified dataset.
109
- # Requires the READER dataset role.
110
- def list_tables dataset_id, options = {}
111
- params = { projectId: @project,
112
- datasetId: dataset_id,
113
- pageToken: options.delete(:token),
114
- maxResults: options.delete(:max)
115
- }.delete_if { |_, v| v.nil? }
116
-
117
- execute(
118
- api_method: @bigquery.tables.list,
119
- parameters: params
120
- )
121
- end
122
-
123
- def get_project_table project_id, dataset_id, table_id
124
- execute(
125
- api_method: @bigquery.tables.get,
126
- parameters: { projectId: project_id, datasetId: dataset_id,
127
- tableId: table_id }
128
- )
129
- end
130
-
131
- ##
132
- # Gets the specified table resource by table ID.
133
- # This method does not return the data in the table,
134
- # it only returns the table resource,
135
- # which describes the structure of this table.
136
- def get_table dataset_id, table_id
137
- get_project_table @project, dataset_id, table_id
138
- end
139
-
140
- ##
141
- # Creates a new, empty table in the dataset.
142
- def insert_table dataset_id, table_id, options = {}
143
- execute(
144
- api_method: @bigquery.tables.insert,
145
- parameters: { projectId: @project, datasetId: dataset_id },
146
- body_object: insert_table_request(dataset_id, table_id, options)
147
- )
148
- end
149
-
150
- ##
151
- # Updates information in an existing table, replacing fields that
152
- # are provided in the submitted table resource.
153
- def patch_table dataset_id, table_id, options = {}
154
- execute(
155
- api_method: @bigquery.tables.patch,
156
- parameters: { projectId: @project, datasetId: dataset_id,
157
- tableId: table_id },
158
- body_object: patch_table_request(options)
159
- )
160
- end
161
-
162
- ##
163
- # Deletes the table specified by tableId from the dataset.
164
- # If the table contains data, all the data will be deleted.
165
- def delete_table dataset_id, table_id
166
- execute(
167
- api_method: @bigquery.tables.delete,
168
- parameters: { projectId: @project, datasetId: dataset_id,
169
- tableId: table_id }
170
- )
171
- end
172
-
173
- ##
174
- # Retrieves data from the table.
175
- def list_tabledata dataset_id, table_id, options = {}
176
- params = { projectId: @project,
177
- datasetId: dataset_id, tableId: table_id,
178
- pageToken: options.delete(:token),
179
- maxResults: options.delete(:max),
180
- startIndex: options.delete(:start)
181
- }.delete_if { |_, v| v.nil? }
182
-
183
- execute(
184
- api_method: @bigquery.tabledata.list,
185
- parameters: params
186
- )
187
- end
188
-
189
- def insert_tabledata dataset_id, table_id, rows, options = {}
190
- execute(
191
- api_method: @bigquery.tabledata.insert_all,
192
- parameters: { projectId: @project,
193
- datasetId: dataset_id,
194
- tableId: table_id },
195
- body_object: insert_tabledata_rows(rows, options)
196
- )
197
- end
198
-
199
- ##
200
- # Lists all jobs in the specified project to which you have
201
- # been granted the READER job role.
202
- def list_jobs options = {}
203
- execute(
204
- api_method: @bigquery.jobs.list,
205
- parameters: list_jobs_params(options)
206
- )
207
- end
208
-
209
- ##
210
- # Returns the job specified by jobID.
211
- def get_job job_id
212
- execute(
213
- api_method: @bigquery.jobs.get,
214
- parameters: { projectId: @project, jobId: job_id }
215
- )
216
- end
217
-
218
- def insert_job config
219
- execute(
220
- api_method: @bigquery.jobs.insert,
221
- parameters: { projectId: @project },
222
- body_object: { "configuration" => config }
223
- )
224
- end
225
-
226
- def query_job query, options = {}
227
- execute(
228
- api_method: @bigquery.jobs.insert,
229
- parameters: { projectId: @project },
230
- body_object: query_table_config(query, options)
231
- )
232
- end
233
-
234
- def query query, options = {}
235
- execute(
236
- api_method: @bigquery.jobs.query,
237
- parameters: { projectId: @project },
238
- body_object: query_config(query, options)
239
- )
240
- end
241
-
242
- ##
243
- # Returns the query data for the job
244
- def job_query_results job_id, options = {}
245
- params = { projectId: @project, jobId: job_id,
246
- pageToken: options.delete(:token),
247
- maxResults: options.delete(:max),
248
- startIndex: options.delete(:start),
249
- timeoutMs: options.delete(:timeout)
250
- }.delete_if { |_, v| v.nil? }
251
-
252
- execute(
253
- api_method: @bigquery.jobs.get_query_results,
254
- parameters: params
255
- )
256
- end
257
-
258
- def copy_table source, target, options = {}
259
- execute(
260
- api_method: @bigquery.jobs.insert,
261
- parameters: { projectId: @project },
262
- body_object: copy_table_config(source, target, options)
263
- )
264
- end
265
-
266
- def link_table table, urls, options = {}
267
- execute(
268
- api_method: @bigquery.jobs.insert,
269
- parameters: { projectId: @project },
270
- body_object: link_table_config(table, urls, options)
271
- )
272
- end
273
-
274
- def extract_table table, storage_files, options = {}
275
- execute(
276
- api_method: @bigquery.jobs.insert,
277
- parameters: { projectId: @project },
278
- body_object: extract_table_config(table, storage_files, options)
279
- )
280
- end
281
-
282
- def load_table table, storage_url, options = {}
283
- execute(
284
- api_method: @bigquery.jobs.insert,
285
- parameters: { projectId: @project },
286
- body_object: load_table_config(table, storage_url,
287
- Array(storage_url).first, options)
288
- )
289
- end
290
-
291
- def load_multipart table, file, options = {}
292
- media = load_media file
293
-
294
- execute(
295
- api_method: @bigquery.jobs.insert,
296
- media: media,
297
- parameters: { projectId: @project, uploadType: "multipart" },
298
- body_object: load_table_config(table, nil, file, options)
299
- )
300
- end
301
-
302
- def load_resumable table, file, chunk_size = nil, options = {}
303
- media = load_media file, chunk_size
304
-
305
- result = execute(
306
- api_method: @bigquery.jobs.insert,
307
- media: media,
308
- parameters: { projectId: @project, uploadType: "resumable" },
309
- body_object: load_table_config(table, nil, file, options)
310
- )
311
- upload = result.resumable_upload
312
- result = execute upload while upload.resumable?
313
- result
314
- end
315
-
316
- def default_access_rules
317
- [
318
- { "role" => "OWNER", "specialGroup" => "projectOwners" },
319
- { "role" => "WRITER", "specialGroup" => "projectWriters" },
320
- { "role" => "READER", "specialGroup" => "projectReaders" },
321
- { "role" => "OWNER", "userByEmail" => credentials.issuer }
322
- ]
323
- end
324
-
325
- ##
326
- # Extracts at least `tbl` group, and possibly `dts` and `prj` groups,
327
- # from strings in the formats: "my_table", "my_dataset.my_table", or
328
- # "my-project:my_dataset.my_table". Then merges project_id and
329
- # dataset_id from the default table if they are missing.
330
- def self.table_ref_from_s str, default_table_ref
331
- str = str.to_s
332
- m = /\A(((?<prj>\S*):)?(?<dts>\S*)\.)?(?<tbl>\S*)\z/.match str
333
- unless m
334
- fail ArgumentError, "unable to identify table from #{str.inspect}"
335
- end
336
- str_table_ref = { "projectId" => m["prj"],
337
- "datasetId" => m["dts"],
338
- "tableId" => m["tbl"] }.delete_if { |_, v| v.nil? }
339
- default_table_ref.merge str_table_ref
340
- end
341
-
342
- def inspect
343
- "#{self.class}(#{@project})"
344
- end
345
-
346
- protected
347
-
348
- ##
349
- # Create the HTTP body for insert dataset
350
- def insert_dataset_request dataset_id, options = {}
351
- {
352
- "kind" => "bigquery#dataset",
353
- "datasetReference" => {
354
- "projectId" => @project,
355
- "datasetId" => dataset_id },
356
- "friendlyName" => options[:name],
357
- "description" => options[:description],
358
- "defaultTableExpirationMs" => options[:expiration],
359
- "access" => options[:access],
360
- "location" => options[:location]
361
- }.delete_if { |_, v| v.nil? }
362
- end
363
-
364
- def patch_dataset_request options = {}
365
- {
366
- friendlyName: options[:name],
367
- description: options[:description],
368
- defaultTableExpirationMs: options[:default_expiration],
369
- access: options[:access]
370
- }.delete_if { |_, v| v.nil? }
371
- end
372
-
373
- ##
374
- # The parameters for the list_jobs call.
375
- def list_jobs_params options = {}
376
- params = { projectId: @project,
377
- allUsers: options.delete(:all),
378
- pageToken: options.delete(:token),
379
- maxResults: options.delete(:max),
380
- stateFilter: options.delete(:filter),
381
- projection: "full"
382
- }.delete_if { |_, v| v.nil? }
383
- params
384
- end
385
-
386
- ##
387
- # Create the HTTP body for insert table
388
- def insert_table_request dataset_id, table_id, options = {}
389
- hash = {
390
- tableReference: {
391
- projectId: @project, datasetId: dataset_id, tableId: table_id
392
- },
393
- friendlyName: options[:name],
394
- description: options[:description],
395
- schema: options[:schema]
396
- }.delete_if { |_, v| v.nil? }
397
- hash["view"] = { "query" => options[:query] } if options[:query]
398
- hash
399
- end
400
-
401
- def patch_table_request options = {}
402
- body = { friendlyName: options[:name],
403
- description: options[:description],
404
- schema: options[:schema]
405
- }.delete_if { |_, v| v.nil? }
406
- body["view"] = { "query" => options[:query] } if options[:query]
407
- body
408
- end
409
-
410
- def insert_tabledata_rows rows, options = {}
411
- {
412
- "kind" => "bigquery#tableDataInsertAllRequest",
413
- "skipInvalidRows" => options[:skip_invalid],
414
- "ignoreUnknownValues" => options[:ignore_unknown],
415
- "rows" => rows.map do |row|
416
- { "insertId" => Digest::MD5.base64digest(row.inspect),
417
- "json" => row }
418
- end
419
- }.delete_if { |_, v| v.nil? }
420
- end
421
-
422
- # rubocop:disable all
423
- # Disabled rubocop because the API is verbose and so these methods
424
- # are going to be verbose.
425
-
426
- ##
427
- # Job description for query job
428
- def query_table_config query, options
429
- dest_table = nil
430
- if options[:table]
431
- dest_table = { "projectId" => options[:table].project_id,
432
- "datasetId" => options[:table].dataset_id,
433
- "tableId" => options[:table].table_id }
434
- end
435
- default_dataset = nil
436
- if dataset = options[:dataset]
437
- if dataset.respond_to? :dataset_id
438
- default_dataset = { "projectId" => dataset.project_id,
439
- "datasetId" => dataset.dataset_id }
440
- else
441
- default_dataset = { "datasetId" => dataset }
442
- end
443
- end
444
- {
445
- "configuration" => {
446
- "query" => {
447
- "query" => query,
448
- # "tableDefinitions" => { ... },
449
- "priority" => priority_value(options[:priority]),
450
- "useQueryCache" => options[:cache],
451
- "destinationTable" => dest_table,
452
- "createDisposition" => create_disposition(options[:create]),
453
- "writeDisposition" => write_disposition(options[:write]),
454
- "allowLargeResults" => options[:large_results],
455
- "flattenResults" => options[:flatten],
456
- "defaultDataset" => default_dataset
457
- }.delete_if { |_, v| v.nil? }
458
- }.delete_if { |_, v| v.nil? }
459
- }
460
- end
461
-
462
- def query_config query, options = {}
463
- dataset_config = nil
464
- dataset_config = {
465
- "datasetId" => options[:dataset],
466
- "projectId" => options[:project] || @project
467
- } if options[:dataset]
468
-
469
- {
470
- "kind" => "bigquery#queryRequest",
471
- "query" => query,
472
- "maxResults" => options[:max],
473
- "defaultDataset" => dataset_config,
474
- "timeoutMs" => options[:timeout],
475
- "dryRun" => options[:dryrun],
476
- "useQueryCache" => options[:cache]
477
- }.delete_if { |_, v| v.nil? }
478
- end
479
-
480
- ##
481
- # Job description for copy job
482
- def copy_table_config source, target, options = {}
483
- {
484
- "configuration" => {
485
- "copy" => {
486
- "sourceTable" => source,
487
- "destinationTable" => target,
488
- "createDisposition" => create_disposition(options[:create]),
489
- "writeDisposition" => write_disposition(options[:write])
490
- }.delete_if { |_, v| v.nil? },
491
- "dryRun" => options[:dryrun]
492
- }.delete_if { |_, v| v.nil? }
493
- }
494
- end
495
-
496
- def link_table_config table, urls, options = {}
497
- path = Array(urls).first
498
- {
499
- "configuration" => {
500
- "link" => {
501
- "sourceUri" => Array(urls),
502
- "destinationTable" => table,
503
- "createDisposition" => create_disposition(options[:create]),
504
- "writeDisposition" => write_disposition(options[:write]),
505
- "sourceFormat" => source_format(path, options[:format])
506
- }.delete_if { |_, v| v.nil? },
507
- "dryRun" => options[:dryrun]
508
- }.delete_if { |_, v| v.nil? }
509
- }
510
- end
511
-
512
- def extract_table_config table, storage_files, options = {}
513
- storage_urls = Array(storage_files).map do |url|
514
- url.respond_to?(:to_gs_url) ? url.to_gs_url : url
515
- end
516
- dest_format = source_format storage_urls.first, options[:format]
517
- {
518
- "configuration" => {
519
- "extract" => {
520
- "destinationUris" => Array(storage_urls),
521
- "sourceTable" => table,
522
- "destinationFormat" => dest_format,
523
- "compression" => options[:compression],
524
- "fieldDelimiter" => options[:delimiter],
525
- "printHeader" => options[:header]
526
- }.delete_if { |_, v| v.nil? },
527
- "dryRun" => options[:dryrun]
528
- }.delete_if { |_, v| v.nil? }
529
- }
530
- end
531
-
532
- def load_table_config table, urls, file, options = {}
533
- path = Array(urls).first
534
- path = Pathname(file).to_path unless file.nil?
535
- {
536
- "configuration" => {
537
- "load" => {
538
- "sourceUris" => Array(urls),
539
- "destinationTable" => table,
540
- "createDisposition" => create_disposition(options[:create]),
541
- "writeDisposition" => write_disposition(options[:write]),
542
- "sourceFormat" => source_format(path, options[:format]),
543
- "projectionFields" => projection_fields(options[:projection_fields]),
544
- "allowJaggedRows" => options[:jagged_rows],
545
- "allowQuotedNewlines" => options[:quoted_newlines],
546
- "encoding" => options[:encoding],
547
- "fieldDelimiter" => options[:delimiter],
548
- "ignoreUnknownValues" => options[:ignore_unknown],
549
- "maxBadRecords" => options[:max_bad_records],
550
- "quote" => options[:quote],
551
- "schema" => options[:schema],
552
- "skipLeadingRows" => options[:skip_leading]
553
- }.delete_if { |_, v| v.nil? },
554
- "dryRun" => options[:dryrun]
555
- }.delete_if { |_, v| v.nil? }
556
- }
557
- end
558
-
559
- def create_disposition str
560
- { "create_if_needed" => "CREATE_IF_NEEDED",
561
- "createifneeded" => "CREATE_IF_NEEDED",
562
- "if_needed" => "CREATE_IF_NEEDED",
563
- "needed" => "CREATE_IF_NEEDED",
564
- "create_never" => "CREATE_NEVER",
565
- "createnever" => "CREATE_NEVER",
566
- "never" => "CREATE_NEVER" }[str.to_s.downcase]
567
- end
568
-
569
- def write_disposition str
570
- { "write_truncate" => "WRITE_TRUNCATE",
571
- "writetruncate" => "WRITE_TRUNCATE",
572
- "truncate" => "WRITE_TRUNCATE",
573
- "write_append" => "WRITE_APPEND",
574
- "writeappend" => "WRITE_APPEND",
575
- "append" => "WRITE_APPEND",
576
- "write_empty" => "WRITE_EMPTY",
577
- "writeempty" => "WRITE_EMPTY",
578
- "empty" => "WRITE_EMPTY" }[str.to_s.downcase]
579
- end
580
-
581
- def priority_value str
582
- { "batch" => "BATCH",
583
- "interactive" => "INTERACTIVE" }[str.to_s.downcase]
584
- end
585
-
586
- def source_format path, format
587
- val = { "csv" => "CSV",
588
- "json" => "NEWLINE_DELIMITED_JSON",
589
- "newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
590
- "avro" => "AVRO",
591
- "datastore" => "DATASTORE_BACKUP",
592
- "datastore_backup" => "DATASTORE_BACKUP"}[format.to_s.downcase]
593
- return val unless val.nil?
594
- return nil if path.nil?
595
- return "CSV" if path.end_with? ".csv"
596
- return "NEWLINE_DELIMITED_JSON" if path.end_with? ".json"
597
- return "AVRO" if path.end_with? ".avro"
598
- return "DATASTORE_BACKUP" if path.end_with? ".backup_info"
599
- nil
600
- end
601
-
602
- def projection_fields array_or_str
603
- Array(array_or_str) unless array_or_str.nil?
604
- end
605
-
606
- # rubocop:enable all
607
-
608
- def load_media file, chunk_size = nil
609
- local_path = Pathname(file).to_path
610
- mime_type = "application/octet-stream"
611
-
612
- media = Google::APIClient::UploadIO.new local_path, mime_type
613
- media.chunk_size = chunk_size unless chunk_size.nil?
614
- media
615
- end
616
-
617
- def execute options
618
- Gcloud::Backoff.new.execute_gapi do
619
- @client.execute options
620
- end
621
- end
622
- end
623
- end
624
- end