gcloud 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +8 -8
  2. data/CHANGELOG.md +36 -0
  3. data/lib/gcloud/backoff.rb +5 -5
  4. data/lib/gcloud/bigquery.rb +24 -0
  5. data/lib/gcloud/bigquery/connection.rb +32 -25
  6. data/lib/gcloud/bigquery/data.rb +99 -1
  7. data/lib/gcloud/bigquery/dataset.rb +5 -13
  8. data/lib/gcloud/bigquery/dataset/list.rb +124 -2
  9. data/lib/gcloud/bigquery/job/list.rb +125 -2
  10. data/lib/gcloud/bigquery/project.rb +30 -27
  11. data/lib/gcloud/bigquery/query_data.rb +102 -1
  12. data/lib/gcloud/bigquery/table.rb +17 -2
  13. data/lib/gcloud/bigquery/table/list.rb +132 -3
  14. data/lib/gcloud/datastore.rb +30 -19
  15. data/lib/gcloud/datastore/dataset.rb +2 -22
  16. data/lib/gcloud/datastore/dataset/lookup_results.rb +160 -4
  17. data/lib/gcloud/datastore/dataset/query_results.rb +229 -23
  18. data/lib/gcloud/datastore/transaction.rb +2 -5
  19. data/lib/gcloud/dns.rb +20 -0
  20. data/lib/gcloud/dns/change/list.rb +109 -6
  21. data/lib/gcloud/dns/connection.rb +18 -9
  22. data/lib/gcloud/dns/project.rb +4 -8
  23. data/lib/gcloud/dns/record/list.rb +96 -13
  24. data/lib/gcloud/dns/zone.rb +9 -24
  25. data/lib/gcloud/dns/zone/list.rb +102 -5
  26. data/lib/gcloud/dns/zone/transaction.rb +1 -1
  27. data/lib/gcloud/logging.rb +19 -0
  28. data/lib/gcloud/logging/entry/list.rb +83 -14
  29. data/lib/gcloud/logging/metric/list.rb +89 -12
  30. data/lib/gcloud/logging/project.rb +18 -30
  31. data/lib/gcloud/logging/resource_descriptor/list.rb +105 -6
  32. data/lib/gcloud/logging/sink/list.rb +89 -12
  33. data/lib/gcloud/pubsub.rb +23 -0
  34. data/lib/gcloud/pubsub/project.rb +21 -29
  35. data/lib/gcloud/pubsub/service.rb +1 -3
  36. data/lib/gcloud/pubsub/subscription/list.rb +167 -13
  37. data/lib/gcloud/pubsub/topic.rb +15 -13
  38. data/lib/gcloud/pubsub/topic/batch.rb +10 -4
  39. data/lib/gcloud/pubsub/topic/list.rb +134 -8
  40. data/lib/gcloud/resource_manager.rb +24 -0
  41. data/lib/gcloud/resource_manager/connection.rb +18 -9
  42. data/lib/gcloud/resource_manager/manager.rb +7 -4
  43. data/lib/gcloud/resource_manager/project/list.rb +93 -14
  44. data/lib/gcloud/storage.rb +63 -0
  45. data/lib/gcloud/storage/bucket.rb +100 -61
  46. data/lib/gcloud/storage/bucket/list.rb +132 -8
  47. data/lib/gcloud/storage/connection.rb +68 -44
  48. data/lib/gcloud/storage/errors.rb +9 -3
  49. data/lib/gcloud/storage/file.rb +48 -4
  50. data/lib/gcloud/storage/file/list.rb +151 -15
  51. data/lib/gcloud/storage/file/verifier.rb +3 -3
  52. data/lib/gcloud/storage/project.rb +15 -30
  53. data/lib/gcloud/translate.rb +20 -0
  54. data/lib/gcloud/translate/connection.rb +12 -3
  55. data/lib/gcloud/version.rb +1 -1
  56. data/lib/gcloud/vision.rb +20 -0
  57. data/lib/gcloud/vision/connection.rb +10 -1
  58. data/lib/gcloud/vision/image.rb +15 -18
  59. metadata +16 -2
@@ -215,30 +215,23 @@ module Gcloud
215
215
  # puts subscription.name
216
216
  # end
217
217
  #
218
- # @example With pagination: (See {Subscription::List#token})
218
+ # @example Retrieve all subscriptions: (See {Subscription::List#all})
219
219
  # require "gcloud"
220
220
  #
221
221
  # gcloud = Gcloud.new
222
222
  # pubsub = gcloud.pubsub
223
223
  #
224
224
  # topic = pubsub.topic "my-topic"
225
- # all_subs = []
226
- # tmp_subs = topic.subscriptions
227
- # while tmp_subs.any? do
228
- # tmp_subs.each do |subscription|
229
- # all_subs << subscription
230
- # end
231
- # # break loop if no more subscriptions available
232
- # break if tmp_subs.token.nil?
233
- # # get the next group of subscriptions
234
- # tmp_subs = topic.subscriptions token: tmp_subs.token
225
+ # subscription = topic.subscriptions
226
+ # subscriptions.all do |subscription|
227
+ # puts subscription.name
235
228
  # end
236
229
  #
237
230
  def subscriptions token: nil, max: nil
238
231
  ensure_service!
239
232
  options = { token: token, max: max }
240
233
  grpc = service.list_topics_subscriptions name, options
241
- Subscription::List.from_grpc grpc, service
234
+ Subscription::List.from_topic_grpc grpc, service, name, max
242
235
  rescue GRPC::BadStatus => e
243
236
  raise Error.from_error(e)
244
237
  end
@@ -248,7 +241,7 @@ module Gcloud
248
241
  ##
249
242
  # Publishes one or more messages to the topic.
250
243
  #
251
- # @param [String] data The message data.
244
+ # @param [String, File] data The message data.
252
245
  # @param [Hash] attributes Optional attributes for the message.
253
246
  # @yield [batch] a block for publishing multiple messages in one request
254
247
  # @yieldparam [Topic::Batch] batch the batch object
@@ -266,6 +259,15 @@ module Gcloud
266
259
  # topic = pubsub.topic "my-topic"
267
260
  # msg = topic.publish "new-message"
268
261
  #
262
+ # @example A message can be published using a File object:
263
+ # require "gcloud"
264
+ #
265
+ # gcloud = Gcloud.new
266
+ # pubsub = gcloud.pubsub
267
+ #
268
+ # topic = pubsub.topic "my-topic"
269
+ # msg = topic.publish File.open("message.txt")
270
+ #
269
271
  # @example Additionally, a message can be published with attributes:
270
272
  # require "gcloud"
271
273
  #
@@ -38,6 +38,13 @@ module Gcloud
38
38
  # All messages added will be published at once.
39
39
  # See {Gcloud::Pubsub::Topic#publish}
40
40
  def publish data, attributes = {}
41
+ # Convert IO-ish objects to strings
42
+ if data.respond_to?(:read) && data.respond_to?(:rewind)
43
+ data.rewind
44
+ data = data.read
45
+ end
46
+ # Convert data to encoded byte array to match the protobuf definition
47
+ data = String(data).force_encoding("ASCII-8BIT")
41
48
  # Convert attributes to strings to match the protobuf definition
42
49
  attributes = Hash[attributes.map { |k, v| [String(k), String(v)] }]
43
50
  @messages << [data, attributes]
@@ -47,10 +54,9 @@ module Gcloud
47
54
  # @private Create Message objects with message ids.
48
55
  def to_gcloud_messages message_ids
49
56
  msgs = @messages.zip(Array(message_ids)).map do |arr, id|
50
- Message.from_grpc(Google::Pubsub::V1::PubsubMessage.new(
51
- data: String(arr[0]).encode("ASCII-8BIT"),
52
- attributes: arr[1],
53
- message_id: id))
57
+ Message.from_grpc(
58
+ Google::Pubsub::V1::PubsubMessage.new(
59
+ data: arr[0], attributes: arr[1], message_id: id))
54
60
  end
55
61
  # Return just one Message if a single publish,
56
62
  # otherwise return the array of Messages.
@@ -28,21 +28,147 @@ module Gcloud
28
28
  attr_accessor :token
29
29
 
30
30
  ##
31
- # Create a new Topic::List with an array of values.
32
- def initialize arr = [], token = nil
31
+ # @private Create a new Topic::List with an array of values.
32
+ def initialize arr = []
33
33
  super arr
34
- @token = token
35
- @token = nil if @token == ""
34
+ end
35
+
36
+ ##
37
+ # Whether there a next page of topics.
38
+ #
39
+ # @return [Boolean]
40
+ #
41
+ # @example
42
+ # require "gcloud"
43
+ #
44
+ # gcloud = Gcloud.new
45
+ # pubsub = gcloud.pubsub
46
+ #
47
+ # topics = pubsub.topics
48
+ # if topics.next?
49
+ # next_topics = topics.next
50
+ # end
51
+ #
52
+ def next?
53
+ !token.nil?
54
+ end
55
+
56
+ ##
57
+ # Retrieve the next page of topics.
58
+ #
59
+ # @return [Topic::List]
60
+ #
61
+ # @example
62
+ # require "gcloud"
63
+ #
64
+ # gcloud = Gcloud.new
65
+ # pubsub = gcloud.pubsub
66
+ #
67
+ # topics = pubsub.topics
68
+ # if topics.next?
69
+ # next_topics = topics.next
70
+ # end
71
+ #
72
+ def next
73
+ return nil unless next?
74
+ ensure_service!
75
+ options = { token: token, max: @max }
76
+ grpc = @service.list_topics options
77
+ self.class.from_grpc grpc, @service, @max
78
+ rescue GRPC::BadStatus => e
79
+ raise Error.from_error(e)
80
+ end
81
+
82
+ ##
83
+ # Retrieves all topics by repeatedly loading {#next} until {#next?}
84
+ # returns `false`. Calls the given block once for each topic, which is
85
+ # passed as the parameter.
86
+ #
87
+ # An Enumerator is returned if no block is given.
88
+ #
89
+ # This method may make several API calls until all topics are retrieved.
90
+ # Be sure to use as narrow a search criteria as possible. Please use
91
+ # with caution.
92
+ #
93
+ # @param [Integer] request_limit The upper limit of API requests to make
94
+ # to load all topics. Default is no limit.
95
+ # @yield [topic] The block for accessing each topic.
96
+ # @yieldparam [Topic] topic The topic object.
97
+ #
98
+ # @return [Enumerator]
99
+ #
100
+ # @example Iterating each topic by passing a block:
101
+ # require "gcloud"
102
+ #
103
+ # gcloud = Gcloud.new
104
+ # pubsub = gcloud.pubsub
105
+ #
106
+ # topics = pubsub.topics
107
+ # topics.all do |topic|
108
+ # puts topic.name
109
+ # end
110
+ #
111
+ # @example Using the enumerator by not passing a block:
112
+ # require "gcloud"
113
+ #
114
+ # gcloud = Gcloud.new
115
+ # pubsub = gcloud.pubsub
116
+ #
117
+ # topics = pubsub.topics
118
+ # all_names = topics.all.map do |topic|
119
+ # topic.name
120
+ # end
121
+ #
122
+ # @example Limit the number of API calls made:
123
+ # require "gcloud"
124
+ #
125
+ # gcloud = Gcloud.new
126
+ # pubsub = gcloud.pubsub
127
+ #
128
+ # topics = pubsub.topics
129
+ # topics.all(request_limit: 10) do |topic|
130
+ # puts topic.name
131
+ # end
132
+ #
133
+ def all request_limit: nil
134
+ request_limit = request_limit.to_i if request_limit
135
+ unless block_given?
136
+ return enum_for(:all, request_limit: request_limit)
137
+ end
138
+ results = self
139
+ loop do
140
+ results.each { |r| yield r }
141
+ if request_limit
142
+ request_limit -= 1
143
+ break if request_limit < 0
144
+ end
145
+ break unless results.next?
146
+ results = results.next
147
+ end
36
148
  end
37
149
 
38
150
  ##
39
151
  # @private New Topic::List from a Google::Pubsub::V1::ListTopicsResponse
40
152
  # object.
41
- def self.from_grpc grpc_list, service
42
- topics = Array(grpc_list.topics).map do |grpc|
153
+ def self.from_grpc grpc_list, service, max = nil
154
+ topics = new(Array(grpc_list.topics).map do |grpc|
43
155
  Topic.from_grpc grpc, service
44
- end
45
- new topics, grpc_list.next_page_token
156
+ end)
157
+ token = grpc_list.next_page_token
158
+ token = nil if token == ""
159
+ topics.instance_variable_set "@token", token
160
+ topics.instance_variable_set "@service", service
161
+ topics.instance_variable_set "@max", max
162
+ topics
163
+ end
164
+
165
+ protected
166
+
167
+ ##
168
+ # @private Raise an error unless an active connection to the service is
169
+ # available.
170
+ def ensure_service!
171
+ fail "Must have active connection to service" unless @service
46
172
  end
47
173
  end
48
174
  end
@@ -201,6 +201,30 @@ module Gcloud
201
201
  # resource_manager.undelete "tokyo-rain-123"
202
202
  # ```
203
203
  #
204
+ # ## Configuring Backoff
205
+ #
206
+ # The {Gcloud::Backoff} class allows users to globally configure how Cloud API
207
+ # requests are automatically retried in the case of some errors, such as a
208
+ # `500` or `503` status code, or a specific internal error code such as
209
+ # `rateLimitExceeded`.
210
+ #
211
+ # If an API call fails, the response will be inspected to see if the call
212
+ # should be retried. If the response matches the criteria, then the request
213
+ # will be retried after a delay. If another error occurs, the delay will be
214
+ # increased incrementally before a subsequent attempt. The first retry will be
215
+ # delayed one second, the second retry two seconds, and so on.
216
+ #
217
+ # ```ruby
218
+ # require "gcloud"
219
+ # require "gcloud/backoff"
220
+ #
221
+ # Gcloud::Backoff.retries = 5 # Raise the maximum number of retries from 3
222
+ # ```
223
+ #
224
+ # See the [Resource Manager error
225
+ # messages](https://cloud.google.com/resource-manager/docs/core_errors)
226
+ # for a list of error conditions.
227
+ #
204
228
  # ## Managing IAM Policies
205
229
  #
206
230
  # Google Cloud Identity and Access Management ([Cloud
@@ -14,6 +14,7 @@
14
14
 
15
15
 
16
16
  require "gcloud/version"
17
+ require "gcloud/backoff"
17
18
  require "google/api_client"
18
19
 
19
20
  module Gcloud
@@ -43,14 +44,14 @@ module Gcloud
43
44
  maxResults: max
44
45
  }.delete_if { |_, v| v.nil? }
45
46
 
46
- @client.execute(
47
+ execute(
47
48
  api_method: @res_man.projects.list,
48
49
  parameters: params
49
50
  )
50
51
  end
51
52
 
52
53
  def get_project project_id
53
- @client.execute(
54
+ execute(
54
55
  api_method: @res_man.projects.get,
55
56
  parameters: { projectId: project_id }
56
57
  )
@@ -60,7 +61,7 @@ module Gcloud
60
61
  project_gapi = { projectId: project_id, name: name,
61
62
  labels: labels }.delete_if { |_, v| v.nil? }
62
63
 
63
- @client.execute(
64
+ execute(
64
65
  api_method: @res_man.projects.create,
65
66
  body_object: project_gapi
66
67
  )
@@ -73,7 +74,7 @@ module Gcloud
73
74
  def update_project project_gapi
74
75
  project_id = project_gapi["projectId"]
75
76
 
76
- @client.execute(
77
+ execute(
77
78
  api_method: @res_man.projects.update,
78
79
  parameters: { projectId: project_id },
79
80
  body_object: project_gapi
@@ -81,28 +82,28 @@ module Gcloud
81
82
  end
82
83
 
83
84
  def delete_project project_id
84
- @client.execute(
85
+ execute(
85
86
  api_method: @res_man.projects.delete,
86
87
  parameters: { projectId: project_id }
87
88
  )
88
89
  end
89
90
 
90
91
  def undelete_project project_id
91
- @client.execute(
92
+ execute(
92
93
  api_method: @res_man.projects.undelete,
93
94
  parameters: { projectId: project_id }
94
95
  )
95
96
  end
96
97
 
97
98
  def get_policy project_id
98
- @client.execute(
99
+ execute(
99
100
  api_method: @res_man.projects.get_iam_policy,
100
101
  parameters: { resource: project_id }
101
102
  )
102
103
  end
103
104
 
104
105
  def set_policy project_id, new_policy
105
- @client.execute(
106
+ execute(
106
107
  api_method: @res_man.projects.set_iam_policy,
107
108
  parameters: { resource: project_id },
108
109
  body_object: { policy: new_policy }
@@ -110,7 +111,7 @@ module Gcloud
110
111
  end
111
112
 
112
113
  def test_permissions project_id, permissions
113
- @client.execute(
114
+ execute(
114
115
  api_method: @res_man.projects.test_iam_permissions,
115
116
  parameters: { resource: project_id },
116
117
  body_object: { permissions: permissions }
@@ -120,6 +121,14 @@ module Gcloud
120
121
  def inspect
121
122
  "#{self.class}(#{@project})"
122
123
  end
124
+
125
+ protected
126
+
127
+ def execute options
128
+ Gcloud::Backoff.new.execute_gapi do
129
+ @client.execute options
130
+ end
131
+ end
123
132
  end
124
133
  end
125
134
  end
@@ -85,6 +85,7 @@ module Gcloud
85
85
  # gcloud = Gcloud.new
86
86
  # resource_manager = gcloud.resource_manager
87
87
  # projects = resource_manager.projects
88
+ #
88
89
  # projects.each do |project|
89
90
  # puts project.project_id
90
91
  # end
@@ -95,24 +96,26 @@ module Gcloud
95
96
  # gcloud = Gcloud.new
96
97
  # resource_manager = gcloud.resource_manager
97
98
  # projects = resource_manager.projects filter: "labels.env:production"
99
+ #
98
100
  # projects.each do |project|
99
101
  # puts project.project_id
100
102
  # end
101
103
  #
102
- # @example With pagination: (See {Gcloud::ResourceManager::Project::List})
104
+ # @example Retrieve all projects: (See {Project::List#all})
103
105
  # require "gcloud"
104
106
  #
105
107
  # gcloud = Gcloud.new
106
108
  # resource_manager = gcloud.resource_manager
107
- # projects = resource_manager.projects.all
108
- # projects.each do |project|
109
+ # projects = resource_manager.projects
110
+ #
111
+ # projects.all do |project|
109
112
  # puts project.project_id
110
113
  # end
111
114
  #
112
115
  def projects filter: nil, token: nil, max: nil
113
116
  resp = connection.list_project filter: filter, token: token, max: max
114
117
  if resp.success?
115
- Project::List.from_response resp, self
118
+ Project::List.from_response resp, self, filter, max
116
119
  else
117
120
  fail ApiError.from_response(resp)
118
121
  end
@@ -25,53 +25,132 @@ module Gcloud
25
25
  attr_accessor :token
26
26
 
27
27
  ##
28
- # Create a new Project::List with an array of Project instances.
28
+ # @private Create a new Project::List with an array of Project
29
+ # instances.
29
30
  def initialize arr = []
30
31
  super arr
31
32
  end
32
33
 
33
34
  ##
34
- # Whether there a next page of projects.
35
+ # Whether there is a next page of projects.
36
+ #
37
+ # @return [Boolean]
38
+ #
39
+ # @example
40
+ # require "gcloud"
41
+ #
42
+ # gcloud = Gcloud.new
43
+ # resource_manager = gcloud.resource_manager
44
+ #
45
+ # projects = resource_manager.projects
46
+ # if projects.next?
47
+ # next_projects = projects.next
48
+ # end
49
+ #
35
50
  def next?
36
51
  !token.nil?
37
52
  end
38
53
 
39
54
  ##
40
55
  # Retrieve the next page of projects.
56
+ #
57
+ # @return [Project::List]
58
+ #
59
+ # @example
60
+ # require "gcloud"
61
+ #
62
+ # gcloud = Gcloud.new
63
+ # resource_manager = gcloud.resource_manager
64
+ #
65
+ # projects = resource_manager.projects
66
+ # if projects.next?
67
+ # next_projects = projects.next
68
+ # end
69
+ #
41
70
  def next
42
71
  return nil unless next?
43
72
  ensure_manager!
44
- @manager.projects token: token
73
+ @manager.projects token: token, filter: @filter, max: @max
45
74
  end
46
75
 
47
76
  ##
48
- # Retrieves all projects by repeatedly loading pages until #next?
49
- # returns false. Returns the list instance for method chaining.
77
+ # Retrieves all projects by repeatedly loading {#next} until {#next?}
78
+ # returns `false`. Calls the given block once for each project, which is
79
+ # passed as the parameter.
50
80
  #
51
- # @example
81
+ # An Enumerator is returned if no block is given.
82
+ #
83
+ # This method may make several API calls until all projects are
84
+ # retrieved. Be sure to use as narrow a search criteria as possible.
85
+ # Please use with caution.
86
+ #
87
+ # @param [Integer] request_limit The upper limit of API requests to make
88
+ # to load all projects. Default is no limit.
89
+ # @yield [project] The block for accessing each project.
90
+ # @yieldparam [Project] project The project object.
91
+ #
92
+ # @return [Enumerator]
93
+ #
94
+ # @example Iterating each project by passing a block:
95
+ # require "gcloud"
96
+ #
97
+ # gcloud = Gcloud.new
98
+ # resource_manager = gcloud.resource_manager
99
+ # projects = resource_manager.projects
100
+ #
101
+ # projects.all do |project|
102
+ # puts project.project_id
103
+ # end
104
+ #
105
+ # @example Using the enumerator by not passing a block:
52
106
  # require "gcloud"
53
107
  #
54
108
  # gcloud = Gcloud.new
55
109
  # resource_manager = gcloud.resource_manager
56
- # projects = resource_manager.projects.all # Load all projects
110
+ # projects = resource_manager.projects
111
+ #
112
+ # all_project_ids = projects.all.map do |project|
113
+ # project.project_id
114
+ # end
57
115
  #
58
- def all
59
- while next?
60
- next_projects = self.next
61
- push(*next_projects)
62
- self.token = next_projects.token
116
+ # @example Limit the number of API calls made:
117
+ # require "gcloud"
118
+ #
119
+ # gcloud = Gcloud.new
120
+ # resource_manager = gcloud.resource_manager
121
+ # projects = resource_manager.projects
122
+ #
123
+ # projects.all(request_limit: 10) do |project|
124
+ # puts project.project_id
125
+ # end
126
+ #
127
+ def all request_limit: nil
128
+ request_limit = request_limit.to_i if request_limit
129
+ unless block_given?
130
+ return enum_for(:all, request_limit: request_limit)
131
+ end
132
+ results = self
133
+ loop do
134
+ results.each { |r| yield r }
135
+ if request_limit
136
+ request_limit -= 1
137
+ break if request_limit < 0
138
+ end
139
+ break unless results.next?
140
+ results = results.next
63
141
  end
64
- self
65
142
  end
66
143
 
67
144
  ##
68
145
  # @private New Projects::List from a response object.
69
- def self.from_response resp, manager
146
+ def self.from_response resp, manager, filter = nil, max = nil
70
147
  projects = new(Array(resp.data["projects"]).map do |gapi_object|
71
148
  Project.from_gapi gapi_object, manager.connection
72
149
  end)
73
150
  projects.instance_variable_set "@token", resp.data["nextPageToken"]
74
151
  projects.instance_variable_set "@manager", manager
152
+ projects.instance_variable_set "@filter", filter
153
+ projects.instance_variable_set "@max", max
75
154
  projects
76
155
  end
77
156