google-cloud-pubsub 1.0.2 → 2.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/AUTHENTICATION.md +16 -54
  3. data/CHANGELOG.md +464 -0
  4. data/CONTRIBUTING.md +328 -116
  5. data/EMULATOR.md +1 -1
  6. data/LOGGING.md +94 -2
  7. data/OVERVIEW.md +121 -68
  8. data/TROUBLESHOOTING.md +2 -8
  9. data/lib/google/cloud/pubsub/acknowledge_result.rb +79 -0
  10. data/lib/google/cloud/pubsub/async_publisher/batch.rb +319 -0
  11. data/lib/google/cloud/pubsub/async_publisher.rb +231 -156
  12. data/lib/google/cloud/pubsub/batch_publisher.rb +60 -30
  13. data/lib/google/cloud/pubsub/convert.rb +33 -7
  14. data/lib/google/cloud/pubsub/credentials.rb +2 -2
  15. data/lib/google/cloud/pubsub/errors.rb +93 -0
  16. data/lib/google/cloud/pubsub/flow_controller.rb +137 -0
  17. data/lib/google/cloud/pubsub/message.rb +45 -4
  18. data/lib/google/cloud/pubsub/policy.rb +3 -2
  19. data/lib/google/cloud/pubsub/project.rb +316 -49
  20. data/lib/google/cloud/pubsub/publish_result.rb +6 -1
  21. data/lib/google/cloud/pubsub/received_message.rb +171 -10
  22. data/lib/google/cloud/pubsub/retry_policy.rb +88 -0
  23. data/lib/google/cloud/pubsub/schema/list.rb +180 -0
  24. data/lib/google/cloud/pubsub/schema.rb +310 -0
  25. data/lib/google/cloud/pubsub/service.rb +285 -269
  26. data/lib/google/cloud/pubsub/snapshot/list.rb +4 -6
  27. data/lib/google/cloud/pubsub/snapshot.rb +5 -2
  28. data/lib/google/cloud/pubsub/subscriber/inventory.rb +69 -32
  29. data/lib/google/cloud/pubsub/subscriber/sequencer.rb +115 -0
  30. data/lib/google/cloud/pubsub/subscriber/stream.rb +108 -49
  31. data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb +191 -30
  32. data/lib/google/cloud/pubsub/subscriber.rb +155 -45
  33. data/lib/google/cloud/pubsub/subscription/list.rb +4 -6
  34. data/lib/google/cloud/pubsub/subscription/push_config.rb +55 -31
  35. data/lib/google/cloud/pubsub/subscription.rb +561 -77
  36. data/lib/google/cloud/pubsub/topic/list.rb +4 -6
  37. data/lib/google/cloud/pubsub/topic.rb +372 -52
  38. data/lib/google/cloud/pubsub/version.rb +1 -1
  39. data/lib/google/cloud/pubsub.rb +35 -46
  40. data/lib/google-cloud-pubsub.rb +21 -27
  41. metadata +26 -189
  42. data/lib/google/cloud/pubsub/v1/credentials.rb +0 -41
  43. data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/iam_policy.rb +0 -21
  44. data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/options.rb +0 -21
  45. data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/policy.rb +0 -21
  46. data/lib/google/cloud/pubsub/v1/doc/google/protobuf/duration.rb +0 -91
  47. data/lib/google/cloud/pubsub/v1/doc/google/protobuf/empty.rb +0 -29
  48. data/lib/google/cloud/pubsub/v1/doc/google/protobuf/field_mask.rb +0 -222
  49. data/lib/google/cloud/pubsub/v1/doc/google/protobuf/timestamp.rb +0 -113
  50. data/lib/google/cloud/pubsub/v1/doc/google/pubsub/v1/pubsub.rb +0 -744
  51. data/lib/google/cloud/pubsub/v1/doc/google/type/expr.rb +0 -19
  52. data/lib/google/cloud/pubsub/v1/publisher_client.rb +0 -786
  53. data/lib/google/cloud/pubsub/v1/publisher_client_config.json +0 -105
  54. data/lib/google/cloud/pubsub/v1/subscriber_client.rb +0 -1385
  55. data/lib/google/cloud/pubsub/v1/subscriber_client_config.json +0 -138
  56. data/lib/google/cloud/pubsub/v1.rb +0 -17
  57. data/lib/google/pubsub/v1/pubsub_pb.rb +0 -249
  58. data/lib/google/pubsub/v1/pubsub_services_pb.rb +0 -211
@@ -14,7 +14,10 @@
14
14
 
15
15
 
16
16
  require "concurrent"
17
+ require "google/cloud/errors"
18
+ require "google/cloud/pubsub/acknowledge_result"
17
19
  require "monitor"
20
+ require "retriable"
18
21
 
19
22
  module Google
20
23
  module Cloud
@@ -25,9 +28,25 @@ module Google
25
28
  class TimedUnaryBuffer
26
29
  include MonitorMixin
27
30
 
28
- attr_reader :max_bytes, :interval
29
-
30
- def initialize subscriber, max_bytes: 500000, interval: 1.0
31
+ attr_reader :max_bytes
32
+ attr_reader :interval
33
+ attr_reader :retry_thread_pool
34
+ attr_reader :callback_thread_pool
35
+
36
+ PERMANENT_FAILURE = "PERMANENT_FAILURE".freeze
37
+ # Google::Cloud::Unavailable error is already retried at gapic level
38
+ EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS = [Google::Cloud::CanceledError,
39
+ Google::Cloud::DeadlineExceededError,
40
+ Google::Cloud::InternalError,
41
+ Google::Cloud::ResourceExhaustedError,
42
+ Google::Cloud::InvalidArgumentError].freeze
43
+ MAX_RETRY_DURATION = 600 # 600s since the server allows ack/modacks for 10 mins max
44
+ MAX_TRIES = 15
45
+ BASE_INTERVAL = 1
46
+ MAX_INTERVAL = 64
47
+ MULTIPLIER = 2
48
+
49
+ def initialize subscriber, max_bytes: 500_000, interval: 1.0
31
50
  super() # to init MonitorMixin
32
51
 
33
52
  @subscriber = subscriber
@@ -39,30 +58,37 @@ module Google
39
58
  # entry.
40
59
  @register = {}
41
60
 
61
+ @ack_callback_register = {}
62
+ @modack_callback_register = {}
63
+
64
+ @retry_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
65
+ @callback_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
42
66
  @task = Concurrent::TimerTask.new execution_interval: interval do
43
67
  flush!
44
68
  end
45
69
  end
46
70
 
47
- def acknowledge ack_ids
71
+ def acknowledge ack_ids, callback = nil
48
72
  return if ack_ids.empty?
49
73
 
50
74
  synchronize do
51
75
  ack_ids.each do |ack_id|
52
76
  # ack has no deadline set, use :ack indicate it is an ack
53
77
  @register[ack_id] = :ack
78
+ @ack_callback_register[ack_id] = callback unless callback.nil?
54
79
  end
55
80
  end
56
81
 
57
82
  true
58
83
  end
59
84
 
60
- def modify_ack_deadline deadline, ack_ids
85
+ def modify_ack_deadline deadline, ack_ids, callback = nil
61
86
  return if ack_ids.empty?
62
87
 
63
88
  synchronize do
64
89
  ack_ids.each do |ack_id|
65
90
  @register[ack_id] = deadline
91
+ @modack_callback_register[ack_id] = callback unless callback.nil?
66
92
  end
67
93
  end
68
94
 
@@ -89,22 +115,46 @@ module Google
89
115
 
90
116
  # Perform the RCP calls concurrently
91
117
  with_threadpool do |pool|
92
- requests[:acknowledge].each do |ack_req|
93
- add_future pool do
94
- @subscriber.service.acknowledge \
95
- ack_req.subscription, *ack_req.ack_ids
118
+ make_acknowledge_request requests, pool
119
+ make_modack_request requests, pool
120
+ end
121
+
122
+ true
123
+ end
124
+
125
+ def make_acknowledge_request requests, pool
126
+ requests[:acknowledge].each do |ack_req|
127
+ add_future pool do
128
+ begin
129
+ @subscriber.service.acknowledge ack_req.subscription, *ack_req.ack_ids
130
+ handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS), ack_req.ack_ids
131
+ rescue *EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS => e
132
+ handle_failure e, ack_req.ack_ids if @subscriber.exactly_once_delivery_enabled
133
+ rescue StandardError => e
134
+ handle_callback construct_result(e), ack_req.ack_ids
96
135
  end
97
136
  end
98
- requests[:modify_ack_deadline].each do |mod_ack_req|
99
- add_future pool do
100
- @subscriber.service.modify_ack_deadline \
101
- mod_ack_req.subscription, mod_ack_req.ack_ids,
102
- mod_ack_req.ack_deadline_seconds
137
+ end
138
+ end
139
+
140
+ def make_modack_request requests, pool
141
+ requests[:modify_ack_deadline].each do |mod_ack_req|
142
+ add_future pool do
143
+ begin
144
+ @subscriber.service.modify_ack_deadline mod_ack_req.subscription, mod_ack_req.ack_ids,
145
+ mod_ack_req.ack_deadline_seconds
146
+ handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS),
147
+ mod_ack_req.ack_ids,
148
+ modack: true
149
+ rescue *EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS => e
150
+ if @subscriber.exactly_once_delivery_enabled
151
+ handle_failure e, mod_ack_req.ack_ids, mod_ack_req.ack_deadline_seconds
152
+ end
153
+ rescue StandardError => e
154
+ handle_callback construct_result(e), mod_ack_req.ack_ids, modack: true
103
155
  end
104
156
  end
105
157
  end
106
-
107
- true
108
158
  end
109
159
 
110
160
  def start
@@ -115,8 +165,9 @@ module Google
115
165
 
116
166
  def stop
117
167
  @task.shutdown
168
+ @retry_thread_pool.shutdown
169
+ @callback_thread_pool.shutdown
118
170
  flush!
119
-
120
171
  self
121
172
  end
122
173
 
@@ -130,6 +181,121 @@ module Google
130
181
 
131
182
  private
132
183
 
184
+ def handle_failure error, ack_ids, ack_deadline_seconds = nil
185
+ error_ack_ids = parse_error error, modack: ack_deadline_seconds.nil?
186
+ unless error_ack_ids.nil?
187
+ handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS),
188
+ ack_ids - error_ack_ids,
189
+ modack: ack_deadline_seconds.nil?
190
+ ack_ids = error_ack_ids
191
+ end
192
+ perform_retry_async ack_ids, ack_deadline_seconds
193
+ end
194
+
195
+ def parse_error error, modack: false
196
+ metadata = error.error_metadata
197
+ return if metadata.nil?
198
+ permanent_failures, temporary_failures = metadata.partition do |_, v|
199
+ v.include? PERMANENT_FAILURE
200
+ end.map(&:to_h)
201
+ unless permanent_failures.empty?
202
+ handle_callback construct_result(error),
203
+ permanent_failures.keys.map(&:to_s),
204
+ modack: modack
205
+ end
206
+ temporary_failures.keys.map(&:to_s) unless temporary_failures.empty?
207
+ end
208
+
209
+ def construct_result error
210
+ case error
211
+ when Google::Cloud::PermissionDeniedError
212
+ AcknowledgeResult.new AcknowledgeResult::PERMISSION_DENIED, error
213
+ when Google::Cloud::FailedPreconditionError
214
+ AcknowledgeResult.new AcknowledgeResult::FAILED_PRECONDITION, error
215
+ when Google::Cloud::InvalidArgumentError
216
+ AcknowledgeResult.new AcknowledgeResult::INVALID_ACK_ID, error
217
+ else
218
+ AcknowledgeResult.new AcknowledgeResult::OTHER, error
219
+ end
220
+ end
221
+
222
+ def handle_callback result, ack_ids, modack: false
223
+ ack_ids.each do |ack_id|
224
+ callback = modack ? @modack_callback_register[ack_id] : @ack_callback_register[ack_id]
225
+ perform_callback_async result, callback unless callback.nil?
226
+ end
227
+ synchronize do
228
+ if modack
229
+ @modack_callback_register.delete_if { |ack_id, _| ack_ids.include? ack_id }
230
+ else
231
+ @ack_callback_register.delete_if { |ack_id, _| ack_ids.include? ack_id }
232
+ end
233
+ end
234
+ end
235
+
236
+ def perform_callback_async result, callback
237
+ return unless retry_thread_pool.running?
238
+ Concurrent::Promises.future_on(
239
+ callback_thread_pool, result, callback, &method(:perform_callback_sync)
240
+ )
241
+ end
242
+
243
+ def perform_callback_sync result, callback
244
+ begin
245
+ callback.call result unless stopped?
246
+ rescue StandardError => e
247
+ @subscriber.error! e
248
+ end
249
+ end
250
+
251
+ def perform_retry_async ack_ids, ack_deadline_seconds = nil
252
+ return unless retry_thread_pool.running?
253
+ Concurrent::Promises.future_on(
254
+ retry_thread_pool, ack_ids, ack_deadline_seconds, &method(:retry_transient_error)
255
+ )
256
+ end
257
+
258
+ def retry_transient_error ack_ids, ack_deadline_seconds
259
+ if ack_deadline_seconds.nil?
260
+ retry_request ack_ids, ack_deadline_seconds.nil? do |retry_ack_ids|
261
+ @subscriber.service.acknowledge subscription_name, *retry_ack_ids
262
+ handle_callback AcknowledgeResult.new AcknowledgeResult::SUCCESS, retry_ack_ids
263
+ end
264
+ else
265
+ retry_request ack_ids, ack_deadline_seconds.nil? do |retry_ack_ids|
266
+ @subscriber.service.modify_ack_deadline subscription_name, retry_ack_ids, ack_deadline_seconds
267
+ handle_callback AcknowledgeResult.new AcknowledgeResult::SUCCESS, retry_ack_ids, modack: true
268
+ end
269
+ end
270
+ end
271
+
272
+ def retry_request ack_ids, modack
273
+ begin
274
+ Retriable.retriable tries: MAX_TRIES,
275
+ base_interval: BASE_INTERVAL,
276
+ max_interval: MAX_INTERVAL,
277
+ multiplier: MULTIPLIER,
278
+ max_elapsed_time: MAX_RETRY_DURATION,
279
+ on: EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS do
280
+ return if ack_ids.nil?
281
+ begin
282
+ yield ack_ids
283
+ rescue Google::Cloud::InvalidArgumentError => e
284
+ error_ack_ids = parse_error e.error_metadata, modack: modack
285
+ unless error_ack_ids.nil?
286
+ handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS),
287
+ ack_ids - error_ack_ids,
288
+ modack: modack
289
+ end
290
+ ack_ids = error_ack_ids
291
+ raise e
292
+ end
293
+ end
294
+ rescue StandardError => e
295
+ handle_callback e, ack_ids, modack: modack
296
+ end
297
+ end
298
+
133
299
  def flush_requests!
134
300
  prev_reg =
135
301
  synchronize do
@@ -140,13 +306,11 @@ module Google
140
306
  end
141
307
 
142
308
  groups = prev_reg.each_pair.group_by { |_ack_id, delay| delay }
143
- req_hash = Hash[groups.map { |k, v| [k, v.map(&:first)] }]
309
+ req_hash = groups.transform_values { |v| v.map(&:first) }
144
310
 
145
311
  requests = { acknowledge: [] }
146
312
  ack_ids = Array(req_hash.delete(:ack)) # ack has no deadline set
147
- if ack_ids.any?
148
- requests[:acknowledge] = create_acknowledge_requests ack_ids
149
- end
313
+ requests[:acknowledge] = create_acknowledge_requests ack_ids if ack_ids.any?
150
314
  requests[:modify_ack_deadline] =
151
315
  req_hash.map do |mod_deadline, mod_ack_ids|
152
316
  create_modify_ack_deadline_requests mod_deadline, mod_ack_ids
@@ -201,8 +365,7 @@ module Google
201
365
  end
202
366
 
203
367
  def with_threadpool
204
- pool = Concurrent::ThreadPoolExecutor.new \
205
- max_threads: @subscriber.push_threads
368
+ pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.push_threads
206
369
 
207
370
  yield pool
208
371
 
@@ -213,18 +376,16 @@ module Google
213
376
  pool.kill
214
377
  begin
215
378
  raise "Timeout making subscriber API calls"
216
- rescue StandardError => error
217
- error! error
379
+ rescue StandardError => e
380
+ error! e
218
381
  end
219
382
  end
220
383
 
221
384
  def add_future pool
222
385
  Concurrent::Promises.future_on pool do
223
- begin
224
- yield
225
- rescue StandardError => error
226
- error! error
227
- end
386
+ yield
387
+ rescue StandardError => e
388
+ error! e
228
389
  end
229
390
  end
230
391
  end
@@ -41,7 +41,7 @@ module Google
41
41
  # subscriber.start
42
42
  #
43
43
  # # Shut down the subscriber when ready to stop receiving messages.
44
- # subscriber.stop.wait!
44
+ # subscriber.stop!
45
45
  #
46
46
  # @attr_reader [String] subscription_name The name of the subscription the
47
47
  # messages are pulled from.
@@ -50,10 +50,10 @@ module Google
50
50
  # @attr_reader [Numeric] deadline The default number of seconds the stream
51
51
  # will hold received messages before modifying the message's ack
52
52
  # deadline. The minimum is 10, the maximum is 600. Default is 60.
53
+ # @attr_reader [Boolean] message_ordering Whether message ordering has
54
+ # been enabled.
53
55
  # @attr_reader [Integer] streams The number of concurrent streams to open
54
- # to pull messages from the subscription. Default is 4.
55
- # @attr_reader [Integer] inventory The number of received messages to be
56
- # collected by subscriber. Default is 1,000.
56
+ # to pull messages from the subscription. Default is 2.
57
57
  # @attr_reader [Integer] callback_threads The number of threads used to
58
58
  # handle the received messages. Default is 8.
59
59
  # @attr_reader [Integer] push_threads The number of threads to handle
@@ -64,32 +64,42 @@ module Google
64
64
  class Subscriber
65
65
  include MonitorMixin
66
66
 
67
- attr_reader :subscription_name, :callback, :deadline, :streams,
68
- :inventory, :callback_threads, :push_threads
67
+ attr_reader :subscription_name
68
+ attr_reader :callback
69
+ attr_reader :deadline
70
+ attr_reader :streams
71
+ attr_reader :message_ordering
72
+ attr_reader :callback_threads
73
+ attr_reader :push_threads
69
74
 
70
75
  ##
71
76
  # @private Implementation attributes.
72
- attr_reader :stream_inventory, :stream_pool, :thread_pool, :buffer,
73
- :service
77
+ attr_reader :stream_pool, :thread_pool, :buffer, :service
78
+
79
+ ##
80
+ # @private Implementation attributes.
81
+ attr_accessor :exactly_once_delivery_enabled
74
82
 
75
83
  ##
76
84
  # @private Create an empty {Subscriber} object.
77
- def initialize subscription_name, callback, deadline: nil, streams: nil,
78
- inventory: nil, threads: {}, service: nil
85
+ def initialize subscription_name, callback, deadline: nil, message_ordering: nil, streams: nil, inventory: nil,
86
+ threads: {}, service: nil
87
+ super() # to init MonitorMixin
88
+
79
89
  @callback = callback
80
90
  @error_callbacks = []
81
91
  @subscription_name = subscription_name
82
92
  @deadline = deadline || 60
83
- @streams = streams || 4
84
- @inventory = inventory || 1000
85
- @callback_threads = (threads[:callback] || 8).to_i
86
- @push_threads = (threads[:push] || 4).to_i
93
+ @streams = streams || 2
94
+ coerce_inventory inventory
95
+ @message_ordering = message_ordering
96
+ @callback_threads = Integer(threads[:callback] || 8)
97
+ @push_threads = Integer(threads[:push] || 4)
98
+ @exactly_once_delivery_enabled = nil
87
99
 
88
- @stream_inventory = @inventory.fdiv(@streams).ceil
89
100
  @service = service
90
101
 
91
- @started = nil
92
- @stopped = nil
102
+ @started = @stopped = nil
93
103
 
94
104
  stream_pool = Array.new @streams do
95
105
  Thread.new { Stream.new self }
@@ -97,8 +107,6 @@ module Google
97
107
  @stream_pool = stream_pool.map(&:value)
98
108
 
99
109
  @buffer = TimedUnaryBuffer.new self
100
-
101
- super() # to init MonitorMixin
102
110
  end
103
111
 
104
112
  ##
@@ -106,6 +114,7 @@ module Google
106
114
  # received messages.
107
115
  #
108
116
  # @return [Subscriber] returns self so calls can be chained.
117
+ #
109
118
  def start
110
119
  start_pool = synchronize do
111
120
  @started = true
@@ -124,27 +133,21 @@ module Google
124
133
 
125
134
  ##
126
135
  # Immediately stops the subscriber. No new messages will be pulled from
127
- # the subscription. All actions taken on received messages that have not
128
- # yet been sent to the API will be sent to the API. All received but
129
- # unprocessed messages will be released back to the API and redelivered.
130
- # Use {#wait!} to block until the subscriber is fully stopped and all
131
- # received messages have been processed or released.
136
+ # the subscription. Use {#wait!} to block until all received messages have
137
+ # been processed or released: All actions taken on received messages that
138
+ # have not yet been sent to the API will be sent to the API. All received
139
+ # but unprocessed messages will be released back to the API and redelivered.
132
140
  #
133
141
  # @return [Subscriber] returns self so calls can be chained.
142
+ #
134
143
  def stop
135
- stop_pool = synchronize do
144
+ synchronize do
136
145
  @started = false
137
146
  @stopped = true
138
-
139
- @stream_pool.map do |stream|
140
- Thread.new { stream.stop }
141
- end
147
+ @stream_pool.map(&:stop)
148
+ wait_stop_buffer_thread!
149
+ self
142
150
  end
143
- stop_pool.map(&:join)
144
- # Stop the buffer after the streams are all stopped
145
- synchronize { @buffer.stop }
146
-
147
- self
148
151
  end
149
152
 
150
153
  ##
@@ -160,14 +163,10 @@ module Google
160
163
  # subscriber is fully stopped. Default will block indefinitely.
161
164
  #
162
165
  # @return [Subscriber] returns self so calls can be chained.
166
+ #
163
167
  def wait! timeout = nil
164
- wait_pool = synchronize do
165
- @stream_pool.map do |stream|
166
- Thread.new { stream.wait! timeout }
167
- end
168
- end
169
- wait_pool.map(&:join)
170
-
168
+ wait_stop_buffer_thread!
169
+ @wait_stop_buffer_thread.join timeout
171
170
  self
172
171
  end
173
172
 
@@ -192,6 +191,7 @@ module Google
192
191
  # Whether the subscriber has been started.
193
192
  #
194
193
  # @return [boolean] `true` when started, `false` otherwise.
194
+ #
195
195
  def started?
196
196
  synchronize { @started }
197
197
  end
@@ -200,6 +200,7 @@ module Google
200
200
  # Whether the subscriber has been stopped.
201
201
  #
202
202
  # @return [boolean] `true` when stopped, `false` otherwise.
203
+ #
203
204
  def stopped?
204
205
  synchronize { @stopped }
205
206
  end
@@ -237,7 +238,7 @@ module Google
237
238
  # subscriber.start
238
239
  #
239
240
  # # Shut down the subscriber when ready to stop receiving messages.
240
- # subscriber.stop.wait!
241
+ # subscriber.stop!
241
242
  #
242
243
  def on_error &block
243
244
  synchronize do
@@ -273,12 +274,91 @@ module Google
273
274
  # subscriber.last_error #=> nil
274
275
  #
275
276
  # # Shut down the subscriber when ready to stop receiving messages.
276
- # subscriber.stop.wait!
277
+ # subscriber.stop!
277
278
  #
278
279
  def last_error
279
280
  synchronize { @last_error }
280
281
  end
281
282
 
283
+ ##
284
+ # The number of received messages to be collected by subscriber. Default is 1,000.
285
+ #
286
+ # @return [Integer] The maximum number of messages.
287
+ #
288
+ def max_outstanding_messages
289
+ @inventory[:max_outstanding_messages]
290
+ end
291
+ # @deprecated Use {#max_outstanding_messages}.
292
+ alias inventory_limit max_outstanding_messages
293
+ # @deprecated Use {#max_outstanding_messages}.
294
+ alias inventory max_outstanding_messages
295
+
296
+ ##
297
+ # The total byte size of received messages to be collected by subscriber. Default is 100,000,000 (100MB).
298
+ #
299
+ # @return [Integer] The maximum number of bytes.
300
+ #
301
+ def max_outstanding_bytes
302
+ @inventory[:max_outstanding_bytes]
303
+ end
304
+ # @deprecated Use {#max_outstanding_bytes}.
305
+ alias inventory_bytesize max_outstanding_bytes
306
+
307
+ ##
308
+ # Whether to enforce flow control at the client side only or to enforce it at both the client and
309
+ # the server. For more details about flow control see https://cloud.google.com/pubsub/docs/pull#config.
310
+ #
311
+ # @return [Boolean] `true` when only client side flow control is enforced, `false` when both client and
312
+ # server side flow control are enforced.
313
+ #
314
+ def use_legacy_flow_control?
315
+ @inventory[:use_legacy_flow_control]
316
+ end
317
+
318
+ ##
319
+ # The number of seconds that received messages can be held awaiting processing. Default is 3,600 (1 hour).
320
+ #
321
+ # @return [Integer] The maximum number of seconds.
322
+ #
323
+ def max_total_lease_duration
324
+ @inventory[:max_total_lease_duration]
325
+ end
326
+ # @deprecated Use {#max_total_lease_duration}.
327
+ alias inventory_extension max_total_lease_duration
328
+
329
+ ##
330
+ # The maximum amount of time in seconds for a single lease extension attempt. Bounds the delay before a message
331
+ # redelivery if the subscriber fails to extend the deadline. Default is 0 (disabled).
332
+ #
333
+ # @return [Integer] The maximum number of seconds.
334
+ #
335
+ def max_duration_per_lease_extension
336
+ @inventory[:max_duration_per_lease_extension]
337
+ end
338
+
339
+ ##
340
+ # The minimum amount of time in seconds for a single lease extension attempt. Bounds the delay before a message
341
+ # redelivery if the subscriber fails to extend the deadline. Default is 0 (disabled).
342
+ #
343
+ # @return [Integer] The minimum number of seconds.
344
+ #
345
+ def min_duration_per_lease_extension
346
+ @inventory[:min_duration_per_lease_extension]
347
+ end
348
+
349
+ ##
350
+ # @private
351
+ def stream_inventory
352
+ {
353
+ limit: @inventory[:max_outstanding_messages].fdiv(@streams).ceil,
354
+ bytesize: @inventory[:max_outstanding_bytes].fdiv(@streams).ceil,
355
+ extension: @inventory[:max_total_lease_duration],
356
+ max_duration_per_lease_extension: @inventory[:max_duration_per_lease_extension],
357
+ min_duration_per_lease_extension: @inventory[:min_duration_per_lease_extension],
358
+ use_legacy_flow_control: @inventory[:use_legacy_flow_control]
359
+ }
360
+ end
361
+
282
362
  # @private returns error object from the stream thread.
283
363
  def error! error
284
364
  error_callbacks = synchronize do
@@ -292,8 +372,7 @@ module Google
292
372
  ##
293
373
  # @private
294
374
  def to_s
295
- "(subscription: #{subscription_name}, " \
296
- "streams: [#{stream_pool.map(&:to_s).join(', ')}])"
375
+ "(subscription: #{subscription_name}, streams: [#{stream_pool.map(&:to_s).join(', ')}])"
297
376
  end
298
377
 
299
378
  ##
@@ -304,6 +383,37 @@ module Google
304
383
 
305
384
  protected
306
385
 
386
+ ##
387
+ # Starts a new thread to call wait! (blocking) on each Stream and then stop the TimedUnaryBuffer.
388
+ def wait_stop_buffer_thread!
389
+ synchronize do
390
+ @wait_stop_buffer_thread ||= Thread.new do
391
+ @stream_pool.map(&:wait!)
392
+ # Shutdown the buffer TimerTask (and flush the buffer) after the streams are all stopped.
393
+ @buffer.stop
394
+ end
395
+ end
396
+ end
397
+
398
+ def coerce_inventory inventory
399
+ @inventory = inventory
400
+ if @inventory.is_a? Hash
401
+ @inventory = @inventory.dup
402
+ # Support deprecated field names
403
+ @inventory[:max_outstanding_messages] ||= @inventory.delete :limit
404
+ @inventory[:max_outstanding_bytes] ||= @inventory.delete :bytesize
405
+ @inventory[:max_total_lease_duration] ||= @inventory.delete :extension
406
+ else
407
+ @inventory = { max_outstanding_messages: @inventory }
408
+ end
409
+ @inventory[:max_outstanding_messages] = Integer(@inventory[:max_outstanding_messages] || 1000)
410
+ @inventory[:max_outstanding_bytes] = Integer(@inventory[:max_outstanding_bytes] || 100_000_000)
411
+ @inventory[:max_total_lease_duration] = Integer(@inventory[:max_total_lease_duration] || 3600)
412
+ @inventory[:max_duration_per_lease_extension] = Integer(@inventory[:max_duration_per_lease_extension] || 0)
413
+ @inventory[:min_duration_per_lease_extension] = Integer(@inventory[:min_duration_per_lease_extension] || 0)
414
+ @inventory[:use_legacy_flow_control] = @inventory[:use_legacy_flow_control] || false
415
+ end
416
+
307
417
  def default_error_callbacks
308
418
  # This is memoized to reduce calls to the configuration.
309
419
  @default_error_callbacks ||= begin
@@ -130,17 +130,15 @@ module Google
130
130
  # puts subscription.name
131
131
  # end
132
132
  #
133
- def all request_limit: nil
133
+ def all request_limit: nil, &block
134
134
  request_limit = request_limit.to_i if request_limit
135
- unless block_given?
136
- return enum_for :all, request_limit: request_limit
137
- end
135
+ return enum_for :all, request_limit: request_limit unless block_given?
138
136
  results = self
139
137
  loop do
140
- results.each { |r| yield r }
138
+ results.each(&block)
141
139
  if request_limit
142
140
  request_limit -= 1
143
- break if request_limit < 0
141
+ break if request_limit.negative?
144
142
  end
145
143
  break unless results.next?
146
144
  results = results.next