google-cloud-pubsub 0.26.0 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,98 @@
1
+ # Copyright 2015 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ module Google
17
+ module Cloud
18
+ module Pubsub
19
+ ##
20
+ # Topic Batch Publisher object used to publish multiple messages at
21
+ # once.
22
+ #
23
+ # @example
24
+ # require "google/cloud/pubsub"
25
+ #
26
+ # pubsub = Google::Cloud::Pubsub.new
27
+ #
28
+ # topic = pubsub.topic "my-topic"
29
+ # msgs = topic.publish do |t|
30
+ # t.publish "task 1 completed", foo: :bar
31
+ # t.publish "task 2 completed", foo: :baz
32
+ # t.publish "task 3 completed", foo: :bif
33
+ # end
34
+ class BatchPublisher
35
+ ##
36
+ # @private The messages to publish
37
+ attr_reader :messages
38
+
39
+ ##
40
+ # @private Create a new instance of the object.
41
+ def initialize data = nil, attributes = {}
42
+ @messages = []
43
+ @mode = :batch
44
+ return if data.nil?
45
+ @mode = :single
46
+ publish data, attributes
47
+ end
48
+
49
+ ##
50
+ # Add a message to the batch to be published to the topic.
51
+ # All messages added to the batch will be published at once.
52
+ # See {Google::Cloud::Pubsub::Topic#publish}
53
+ def publish data, attributes = {}
54
+ @messages << create_pubsub_message(data, attributes)
55
+ end
56
+
57
+ ##
58
+ # @private Create Message objects with message ids.
59
+ def to_gcloud_messages message_ids
60
+ msgs = @messages.zip(Array(message_ids)).map do |msg, id|
61
+ msg.message_id = id
62
+ Message.from_grpc msg
63
+ end
64
+ # Return just one Message if a single publish,
65
+ # otherwise return the array of Messages.
66
+ if @mode == :single && msgs.count <= 1
67
+ msgs.first
68
+ else
69
+ msgs
70
+ end
71
+ end
72
+
73
+ protected
74
+
75
+ def create_pubsub_message data, attributes
76
+ attributes ||= {}
77
+ if data.is_a?(::Hash) && attributes.empty?
78
+ attributes = data
79
+ data = nil
80
+ end
81
+ # Convert IO-ish objects to strings
82
+ if data.respond_to?(:read) && data.respond_to?(:rewind)
83
+ data.rewind
84
+ data = data.read
85
+ end
86
+ # Convert data to encoded byte array to match the protobuf defn
87
+ data_bytes = String(data).dup.force_encoding("ASCII-8BIT").freeze
88
+
89
+ # Convert attributes to strings to match the protobuf definition
90
+ attributes = Hash[attributes.map { |k, v| [String(k), String(v)] }]
91
+
92
+ Google::Pubsub::V1::PubsubMessage.new data: data_bytes,
93
+ attributes: attributes
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,63 @@
1
+ # Copyright 2017 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ require "time"
17
+
18
+ module Google
19
+ module Cloud
20
+ module Pubsub
21
+ ##
22
+ # @private Helper module for converting Pub/Sub values.
23
+ module Convert
24
+ module ClassMethods
25
+ def time_to_timestamp time
26
+ return nil if time.nil?
27
+
28
+ # Force the object to be a Time object.
29
+ time = time.to_time
30
+
31
+ Google::Protobuf::Timestamp.new \
32
+ seconds: time.to_i,
33
+ nanos: time.nsec
34
+ end
35
+
36
+ def timestamp_to_time timestamp
37
+ return nil if timestamp.nil?
38
+
39
+ Time.at timestamp.seconds, Rational(timestamp.nanos, 1000)
40
+ end
41
+
42
+ def number_to_duration number
43
+ return nil if number.nil?
44
+
45
+ Google::Protobuf::Duration.new \
46
+ seconds: number.to_i,
47
+ nanos: (number.remainder(1) * 1000000000).round
48
+ end
49
+
50
+ def duration_to_number duration
51
+ return nil if duration.nil?
52
+
53
+ return duration.seconds if duration.nanos == 0
54
+
55
+ duration.seconds + (duration.nanos / 1000000000.0)
56
+ end
57
+ end
58
+
59
+ extend ClassMethods
60
+ end
61
+ end
62
+ end
63
+ end
@@ -13,6 +13,7 @@
13
13
  # limitations under the License.
14
14
 
15
15
 
16
+ require "google/cloud/pubsub/convert"
16
17
  require "google/cloud/errors"
17
18
 
18
19
  module Google
@@ -36,12 +37,12 @@ module Google
36
37
  # # Publish a message
37
38
  # topic = pubsub.topic "my-topic"
38
39
  # message = topic.publish "task completed"
39
- # message.data #=> "task completed"
40
+ # message.data #=> "task completed"
40
41
  #
41
42
  # # Pull a message
42
43
  # sub = pubsub.subscription "my-topic-sub"
43
44
  # received_message = sub.pull.first
44
- # received_message.message.data #=> "task completed"
45
+ # received_message.message.data #=> "task completed"
45
46
  #
46
47
  class Message
47
48
  ##
@@ -61,13 +62,14 @@ module Google
61
62
  end
62
63
 
63
64
  ##
64
- # The received data.
65
+ # The message payload. This data is a list of bytes encoded as
66
+ # ASCII-8BIT.
65
67
  def data
66
68
  @grpc.data
67
69
  end
68
70
 
69
71
  ##
70
- # The received attributes.
72
+ # Optional attributes for the message.
71
73
  def attributes
72
74
  return @grpc.attributes.to_h if @grpc.attributes.respond_to? :to_h
73
75
  # Enumerable doesn't have to_h on Ruby 2.0, so fallback to this
@@ -82,6 +84,13 @@ module Google
82
84
  end
83
85
  alias_method :msg_id, :message_id
84
86
 
87
+ ##
88
+ # The time at which the message was published.
89
+ def published_at
90
+ Convert.timestamp_to_time @grpc.publish_time
91
+ end
92
+ alias_method :publish_time, :published_at
93
+
85
94
  ##
86
95
  # @private New Message from a Google::Pubsub::V1::PubsubMessage object.
87
96
  def self.from_grpc grpc
@@ -18,6 +18,7 @@ require "google/cloud/env"
18
18
  require "google/cloud/pubsub/service"
19
19
  require "google/cloud/pubsub/credentials"
20
20
  require "google/cloud/pubsub/topic"
21
+ require "google/cloud/pubsub/batch_publisher"
21
22
  require "google/cloud/pubsub/snapshot"
22
23
 
23
24
  module Google
@@ -83,25 +84,36 @@ module Google
83
84
  ##
84
85
  # Retrieves topic by name.
85
86
  #
86
- # The topic will be created if the topic does not exist and the
87
- # `autocreate` option is set to true.
88
- #
89
87
  # @param [String] topic_name Name of a topic.
90
- # @param [Boolean] autocreate Flag to control whether the requested
91
- # topic will be created if it does not exist. Ignored if `skip_lookup`
92
- # is `true`. The default value is `false`.
93
88
  # @param [String] project If the topic belongs to a project other than
94
89
  # the one currently connected to, the alternate project ID can be
95
- # specified here.
90
+ # specified here. Optional.
96
91
  # @param [Boolean] skip_lookup Optionally create a {Topic} object
97
92
  # without verifying the topic resource exists on the Pub/Sub service.
98
93
  # Calls made on this object will raise errors if the topic resource
99
- # does not exist. Default is `false`.
94
+ # does not exist. Default is `false`. Optional.
95
+ # @param [Hash] async A hash of values to configure the topic's
96
+ # {AsyncPublisher} that is created when {Topic#publish_async}
97
+ # is called. Optional.
98
+ #
99
+ # Hash keys and values may include the following:
100
+ #
101
+ # * `:max_bytes` (Integer) The maximum size of messages to be
102
+ # collected before the batch is published. Default is 10,000,000
103
+ # (10MB).
104
+ # * `:max_messages` (Integer) The maximum number of messages to be
105
+ # collected before the batch is published. Default is 1,000.
106
+ # * `:interval` (Numeric) The number of seconds to collect messages
107
+ # before the batch is published. Default is 0.25.
108
+ # * `:threads` (Hash) The number of threads to create to handle
109
+ # concurrent calls by the publisher:
110
+ # * `:publish` (Integer) The number of threads used to publish
111
+ # messages. Default is 4.
112
+ # * `:callback` (Integer) The number of threads to handle the
113
+ # published messages' callbacks. Default is 8.
100
114
  #
101
115
  # @return [Google::Cloud::Pubsub::Topic, nil] Returns `nil` if topic
102
- # does not exist. Will return a newly created{
103
- # Google::Cloud::Pubsub::Topic} if the topic does not exist and
104
- # `autocreate` is set to `true`.
116
+ # does not exist.
105
117
  #
106
118
  # @example
107
119
  # require "google/cloud/pubsub"
@@ -115,12 +127,6 @@ module Google
115
127
  # pubsub = Google::Cloud::Pubsub.new
116
128
  # topic = pubsub.topic "non-existing-topic" # nil
117
129
  #
118
- # @example With the `autocreate` option set to `true`.
119
- # require "google/cloud/pubsub"
120
- #
121
- # pubsub = Google::Cloud::Pubsub.new
122
- # topic = pubsub.topic "non-existing-topic", autocreate: true
123
- #
124
130
  # @example Create topic in a different project with the `project` flag.
125
131
  # require "google/cloud/pubsub"
126
132
  #
@@ -133,14 +139,30 @@ module Google
133
139
  # pubsub = Google::Cloud::Pubsub.new
134
140
  # topic = pubsub.topic "another-topic", skip_lookup: true
135
141
  #
136
- def topic topic_name, autocreate: nil, project: nil, skip_lookup: nil
142
+ # @example Configuring AsyncPublisher to increase concurrent callbacks:
143
+ # require "google/cloud/pubsub"
144
+ #
145
+ # pubsub = Google::Cloud::Pubsub.new
146
+ # topic = pubsub.topic "my-topic",
147
+ # async: { threads: { callback: 16 } }
148
+ #
149
+ # topic.publish_async "task completed" do |result|
150
+ # if result.succeeded?
151
+ # log_publish_success result.data
152
+ # else
153
+ # log_publish_failure result.data, result.error
154
+ # end
155
+ # end
156
+ #
157
+ # topic.async_publisher.stop.wait!
158
+ #
159
+ def topic topic_name, project: nil, skip_lookup: nil, async: nil
137
160
  ensure_service!
138
161
  options = { project: project }
139
162
  return Topic.new_lazy(topic_name, service, options) if skip_lookup
140
163
  grpc = service.get_topic topic_name
141
- Topic.from_grpc grpc, service
164
+ Topic.from_grpc grpc, service, async: async
142
165
  rescue Google::Cloud::NotFoundError
143
- return create_topic(topic_name) if autocreate
144
166
  nil
145
167
  end
146
168
  alias_method :get_topic, :topic
@@ -150,6 +172,25 @@ module Google
150
172
  # Creates a new topic.
151
173
  #
152
174
  # @param [String] topic_name Name of a topic.
175
+ # @param [Hash] async A hash of values to configure the topic's
176
+ # {AsyncPublisher} that is created when {Topic#publish_async}
177
+ # is called. Optional.
178
+ #
179
+ # Hash keys and values may include the following:
180
+ #
181
+ # * `:max_bytes` (Integer) The maximum size of messages to be
182
+ # collected before the batch is published. Default is 10,000,000
183
+ # (10MB).
184
+ # * `:max_messages` (Integer) The maximum number of messages to be
185
+ # collected before the batch is published. Default is 1,000.
186
+ # * `:interval` (Numeric) The number of seconds to collect messages
187
+ # before the batch is published. Default is 0.25.
188
+ # * `:threads` (Hash) The number of threads to create to handle
189
+ # concurrent calls by the publisher:
190
+ # * `:publish` (Integer) The number of threads used to publish
191
+ # messages. Default is 4.
192
+ # * `:callback` (Integer) The number of threads to handle the
193
+ # published messages' callbacks. Default is 8.
153
194
  #
154
195
  # @return [Google::Cloud::Pubsub::Topic]
155
196
  #
@@ -159,10 +200,10 @@ module Google
159
200
  # pubsub = Google::Cloud::Pubsub.new
160
201
  # topic = pubsub.create_topic "my-topic"
161
202
  #
162
- def create_topic topic_name
203
+ def create_topic topic_name, async: nil
163
204
  ensure_service!
164
205
  grpc = service.create_topic topic_name
165
- Topic.from_grpc grpc, service
206
+ Topic.from_grpc grpc, service, async: async
166
207
  end
167
208
  alias_method :new_topic, :create_topic
168
209
 
@@ -206,161 +247,6 @@ module Google
206
247
  alias_method :find_topics, :topics
207
248
  alias_method :list_topics, :topics
208
249
 
209
- ##
210
- # Publishes one or more messages to the given topic. The topic will be
211
- # created if the topic does previously not exist and the `autocreate`
212
- # option is provided.
213
- #
214
- # A note about auto-creating the topic: Any message published to a topic
215
- # without a subscription will be lost.
216
- #
217
- # @param [String] topic_name Name of a topic.
218
- # @param [String, File] data The message data.
219
- # @param [Hash] attributes Optional attributes for the message.
220
- # @option attributes [Boolean] :autocreate Flag to control whether the
221
- # provided topic will be created if it does not exist.
222
- # @yield [publisher] a block for publishing multiple messages in one
223
- # request
224
- # @yieldparam [Topic::Publisher] publisher the topic publisher object
225
- #
226
- # @return [Message, Array<Message>] Returns the published message when
227
- # called without a block, or an array of messages when called with a
228
- # block.
229
- #
230
- # @example
231
- # require "google/cloud/pubsub"
232
- #
233
- # pubsub = Google::Cloud::Pubsub.new
234
- #
235
- # msg = pubsub.publish "my-topic", "task completed"
236
- #
237
- # @example A message can be published using a File object:
238
- # require "google/cloud/pubsub"
239
- #
240
- # pubsub = Google::Cloud::Pubsub.new
241
- #
242
- # msg = pubsub.publish "my-topic", File.open("message.txt")
243
- #
244
- # @example Additionally, a message can be published with attributes:
245
- # require "google/cloud/pubsub"
246
- #
247
- # pubsub = Google::Cloud::Pubsub.new
248
- #
249
- # msg = pubsub.publish "my-topic", "task completed", foo: :bar,
250
- # this: :that
251
- #
252
- # @example Multiple messages can be sent at the same time using a block:
253
- # require "google/cloud/pubsub"
254
- #
255
- # pubsub = Google::Cloud::Pubsub.new
256
- #
257
- # msgs = pubsub.publish "my-topic" do |p|
258
- # p.publish "task 1 completed", foo: :bar
259
- # p.publish "task 2 completed", foo: :baz
260
- # p.publish "task 3 completed", foo: :bif
261
- # end
262
- #
263
- # @example With `autocreate`:
264
- # require "google/cloud/pubsub"
265
- #
266
- # pubsub = Google::Cloud::Pubsub.new
267
- #
268
- # msg = pubsub.publish "new-topic", "task completed", autocreate: true
269
- #
270
- def publish topic_name, data = nil, attributes = {}
271
- # Fix parameters
272
- if data.is_a?(::Hash) && attributes.empty?
273
- attributes = data
274
- data = nil
275
- end
276
- # extract autocreate option
277
- autocreate = attributes.delete :autocreate
278
- ensure_service!
279
- publisher = Topic::Publisher.new data, attributes
280
- yield publisher if block_given?
281
- return nil if publisher.messages.count.zero?
282
- publish_batch_messages topic_name, publisher, autocreate
283
- end
284
-
285
- ##
286
- # Creates a new {Subscription} object for the provided topic. The topic
287
- # will be created if the topic does previously not exist and the
288
- # `autocreate` option is provided.
289
- #
290
- # @param [String] topic_name Name of a topic.
291
- # @param [String] subscription_name Name of the new subscription. Must
292
- # start with a letter, and contain only letters ([A-Za-z]), numbers
293
- # ([0-9], dashes (-), underscores (_), periods (.), tildes (~), plus
294
- # (+) or percent signs (%). It must be between 3 and 255 characters in
295
- # length, and it must not start with "goog".
296
- # @param [Integer] deadline The maximum number of seconds after a
297
- # subscriber receives a message before the subscriber should
298
- # acknowledge the message.
299
- # @param [Boolean] retain_acked Indicates whether to retain acknowledged
300
- # messages. If `true`, then messages are not expunged from the
301
- # subscription's backlog, even if they are acknowledged, until they
302
- # fall out of the `retention_duration` window. Default is `false`.
303
- # @param [Numeric] retention How long to retain unacknowledged messages
304
- # in the subscription's backlog, from the moment a message is
305
- # published. If `retain_acked` is `true`, then this also configures
306
- # the retention of acknowledged messages, and thus configures how far
307
- # back in time a {#seek} can be done. Cannot be more than 604,800
308
- # seconds (7 days) or less than 600 seconds (10 minutes). Default is
309
- # 604,800 seconds (7 days).
310
- # @param [String] endpoint A URL locating the endpoint to which messages
311
- # should be pushed.
312
- # @param [String] autocreate Flag to control whether the topic will be
313
- # created if it does not exist.
314
- #
315
- # @return [Google::Cloud::Pubsub::Subscription]
316
- #
317
- # @example
318
- # require "google/cloud/pubsub"
319
- #
320
- # pubsub = Google::Cloud::Pubsub.new
321
- #
322
- # sub = pubsub.subscribe "my-topic", "my-topic-sub"
323
- # sub.name #=> "my-topic-sub"
324
- #
325
- # @example Wait 2 minutes for acknowledgement and push all to endpoint:
326
- # require "google/cloud/pubsub"
327
- #
328
- # pubsub = Google::Cloud::Pubsub.new
329
- #
330
- # sub = pubsub.subscribe "my-topic", "my-topic-sub",
331
- # deadline: 120,
332
- # endpoint: "https://example.com/push"
333
- #
334
- # @example With `autocreate`:
335
- # require "google/cloud/pubsub"
336
- #
337
- # pubsub = Google::Cloud::Pubsub.new
338
- #
339
- # sub = pubsub.subscribe "new-topic", "new-topic-sub",
340
- # autocreate: true
341
- #
342
- def subscribe topic_name, subscription_name, deadline: nil,
343
- retain_acked: false, retention: nil, endpoint: nil,
344
- autocreate: nil
345
- ensure_service!
346
- options = { deadline: deadline, retain_acked: retain_acked,
347
- retention: retention, endpoint: endpoint }
348
- grpc = service.create_subscription topic_name,
349
- subscription_name, options
350
- Subscription.from_grpc grpc, service
351
- rescue Google::Cloud::NotFoundError => e
352
- if autocreate
353
- create_topic topic_name
354
- return subscribe(topic_name, subscription_name,
355
- deadline: deadline, retain_acked: retain_acked,
356
- retention: retention, endpoint: endpoint,
357
- autocreate: false)
358
- end
359
- raise e
360
- end
361
- alias_method :create_subscription, :subscribe
362
- alias_method :new_subscription, :subscribe
363
-
364
250
  ##
365
251
  # Retrieves subscription by name.
366
252
  #
@@ -497,15 +383,9 @@ module Google
497
383
 
498
384
  ##
499
385
  # Call the publish API with arrays of data data and attrs.
500
- def publish_batch_messages topic_name, batch, autocreate = false
386
+ def publish_batch_messages topic_name, batch
501
387
  grpc = service.publish topic_name, batch.messages
502
388
  batch.to_gcloud_messages Array(grpc.message_ids)
503
- rescue Google::Cloud::NotFoundError => e
504
- if autocreate
505
- create_topic topic_name
506
- return publish_batch_messages topic_name, batch, false
507
- end
508
- raise e
509
389
  end
510
390
  end
511
391
  end