google-cloud-pubsub 1.0.2 → 2.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AUTHENTICATION.md +16 -54
- data/CHANGELOG.md +464 -0
- data/CONTRIBUTING.md +328 -116
- data/EMULATOR.md +1 -1
- data/LOGGING.md +94 -2
- data/OVERVIEW.md +121 -68
- data/TROUBLESHOOTING.md +2 -8
- data/lib/google/cloud/pubsub/acknowledge_result.rb +79 -0
- data/lib/google/cloud/pubsub/async_publisher/batch.rb +319 -0
- data/lib/google/cloud/pubsub/async_publisher.rb +231 -156
- data/lib/google/cloud/pubsub/batch_publisher.rb +60 -30
- data/lib/google/cloud/pubsub/convert.rb +33 -7
- data/lib/google/cloud/pubsub/credentials.rb +2 -2
- data/lib/google/cloud/pubsub/errors.rb +93 -0
- data/lib/google/cloud/pubsub/flow_controller.rb +137 -0
- data/lib/google/cloud/pubsub/message.rb +45 -4
- data/lib/google/cloud/pubsub/policy.rb +3 -2
- data/lib/google/cloud/pubsub/project.rb +316 -49
- data/lib/google/cloud/pubsub/publish_result.rb +6 -1
- data/lib/google/cloud/pubsub/received_message.rb +171 -10
- data/lib/google/cloud/pubsub/retry_policy.rb +88 -0
- data/lib/google/cloud/pubsub/schema/list.rb +180 -0
- data/lib/google/cloud/pubsub/schema.rb +310 -0
- data/lib/google/cloud/pubsub/service.rb +285 -269
- data/lib/google/cloud/pubsub/snapshot/list.rb +4 -6
- data/lib/google/cloud/pubsub/snapshot.rb +5 -2
- data/lib/google/cloud/pubsub/subscriber/inventory.rb +69 -32
- data/lib/google/cloud/pubsub/subscriber/sequencer.rb +115 -0
- data/lib/google/cloud/pubsub/subscriber/stream.rb +108 -49
- data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb +191 -30
- data/lib/google/cloud/pubsub/subscriber.rb +155 -45
- data/lib/google/cloud/pubsub/subscription/list.rb +4 -6
- data/lib/google/cloud/pubsub/subscription/push_config.rb +55 -31
- data/lib/google/cloud/pubsub/subscription.rb +561 -77
- data/lib/google/cloud/pubsub/topic/list.rb +4 -6
- data/lib/google/cloud/pubsub/topic.rb +372 -52
- data/lib/google/cloud/pubsub/version.rb +1 -1
- data/lib/google/cloud/pubsub.rb +35 -46
- data/lib/google-cloud-pubsub.rb +21 -27
- metadata +26 -189
- data/lib/google/cloud/pubsub/v1/credentials.rb +0 -41
- data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/iam_policy.rb +0 -21
- data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/options.rb +0 -21
- data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/policy.rb +0 -21
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/duration.rb +0 -91
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/empty.rb +0 -29
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/field_mask.rb +0 -222
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/timestamp.rb +0 -113
- data/lib/google/cloud/pubsub/v1/doc/google/pubsub/v1/pubsub.rb +0 -744
- data/lib/google/cloud/pubsub/v1/doc/google/type/expr.rb +0 -19
- data/lib/google/cloud/pubsub/v1/publisher_client.rb +0 -786
- data/lib/google/cloud/pubsub/v1/publisher_client_config.json +0 -105
- data/lib/google/cloud/pubsub/v1/subscriber_client.rb +0 -1385
- data/lib/google/cloud/pubsub/v1/subscriber_client_config.json +0 -138
- data/lib/google/cloud/pubsub/v1.rb +0 -17
- data/lib/google/pubsub/v1/pubsub_pb.rb +0 -249
- data/lib/google/pubsub/v1/pubsub_services_pb.rb +0 -211
@@ -14,7 +14,10 @@
|
|
14
14
|
|
15
15
|
|
16
16
|
require "concurrent"
|
17
|
+
require "google/cloud/errors"
|
18
|
+
require "google/cloud/pubsub/acknowledge_result"
|
17
19
|
require "monitor"
|
20
|
+
require "retriable"
|
18
21
|
|
19
22
|
module Google
|
20
23
|
module Cloud
|
@@ -25,9 +28,25 @@ module Google
|
|
25
28
|
class TimedUnaryBuffer
|
26
29
|
include MonitorMixin
|
27
30
|
|
28
|
-
attr_reader :max_bytes
|
29
|
-
|
30
|
-
|
31
|
+
attr_reader :max_bytes
|
32
|
+
attr_reader :interval
|
33
|
+
attr_reader :retry_thread_pool
|
34
|
+
attr_reader :callback_thread_pool
|
35
|
+
|
36
|
+
PERMANENT_FAILURE = "PERMANENT_FAILURE".freeze
|
37
|
+
# Google::Cloud::Unavailable error is already retried at gapic level
|
38
|
+
EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS = [Google::Cloud::CanceledError,
|
39
|
+
Google::Cloud::DeadlineExceededError,
|
40
|
+
Google::Cloud::InternalError,
|
41
|
+
Google::Cloud::ResourceExhaustedError,
|
42
|
+
Google::Cloud::InvalidArgumentError].freeze
|
43
|
+
MAX_RETRY_DURATION = 600 # 600s since the server allows ack/modacks for 10 mins max
|
44
|
+
MAX_TRIES = 15
|
45
|
+
BASE_INTERVAL = 1
|
46
|
+
MAX_INTERVAL = 64
|
47
|
+
MULTIPLIER = 2
|
48
|
+
|
49
|
+
def initialize subscriber, max_bytes: 500_000, interval: 1.0
|
31
50
|
super() # to init MonitorMixin
|
32
51
|
|
33
52
|
@subscriber = subscriber
|
@@ -39,30 +58,37 @@ module Google
|
|
39
58
|
# entry.
|
40
59
|
@register = {}
|
41
60
|
|
61
|
+
@ack_callback_register = {}
|
62
|
+
@modack_callback_register = {}
|
63
|
+
|
64
|
+
@retry_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
|
65
|
+
@callback_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
|
42
66
|
@task = Concurrent::TimerTask.new execution_interval: interval do
|
43
67
|
flush!
|
44
68
|
end
|
45
69
|
end
|
46
70
|
|
47
|
-
def acknowledge ack_ids
|
71
|
+
def acknowledge ack_ids, callback = nil
|
48
72
|
return if ack_ids.empty?
|
49
73
|
|
50
74
|
synchronize do
|
51
75
|
ack_ids.each do |ack_id|
|
52
76
|
# ack has no deadline set, use :ack indicate it is an ack
|
53
77
|
@register[ack_id] = :ack
|
78
|
+
@ack_callback_register[ack_id] = callback unless callback.nil?
|
54
79
|
end
|
55
80
|
end
|
56
81
|
|
57
82
|
true
|
58
83
|
end
|
59
84
|
|
60
|
-
def modify_ack_deadline deadline, ack_ids
|
85
|
+
def modify_ack_deadline deadline, ack_ids, callback = nil
|
61
86
|
return if ack_ids.empty?
|
62
87
|
|
63
88
|
synchronize do
|
64
89
|
ack_ids.each do |ack_id|
|
65
90
|
@register[ack_id] = deadline
|
91
|
+
@modack_callback_register[ack_id] = callback unless callback.nil?
|
66
92
|
end
|
67
93
|
end
|
68
94
|
|
@@ -89,22 +115,46 @@ module Google
|
|
89
115
|
|
90
116
|
# Perform the RCP calls concurrently
|
91
117
|
with_threadpool do |pool|
|
92
|
-
requests
|
93
|
-
|
94
|
-
|
95
|
-
|
118
|
+
make_acknowledge_request requests, pool
|
119
|
+
make_modack_request requests, pool
|
120
|
+
end
|
121
|
+
|
122
|
+
true
|
123
|
+
end
|
124
|
+
|
125
|
+
def make_acknowledge_request requests, pool
|
126
|
+
requests[:acknowledge].each do |ack_req|
|
127
|
+
add_future pool do
|
128
|
+
begin
|
129
|
+
@subscriber.service.acknowledge ack_req.subscription, *ack_req.ack_ids
|
130
|
+
handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS), ack_req.ack_ids
|
131
|
+
rescue *EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS => e
|
132
|
+
handle_failure e, ack_req.ack_ids if @subscriber.exactly_once_delivery_enabled
|
133
|
+
rescue StandardError => e
|
134
|
+
handle_callback construct_result(e), ack_req.ack_ids
|
96
135
|
end
|
97
136
|
end
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
def make_modack_request requests, pool
|
141
|
+
requests[:modify_ack_deadline].each do |mod_ack_req|
|
142
|
+
add_future pool do
|
143
|
+
begin
|
144
|
+
@subscriber.service.modify_ack_deadline mod_ack_req.subscription, mod_ack_req.ack_ids,
|
145
|
+
mod_ack_req.ack_deadline_seconds
|
146
|
+
handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS),
|
147
|
+
mod_ack_req.ack_ids,
|
148
|
+
modack: true
|
149
|
+
rescue *EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS => e
|
150
|
+
if @subscriber.exactly_once_delivery_enabled
|
151
|
+
handle_failure e, mod_ack_req.ack_ids, mod_ack_req.ack_deadline_seconds
|
152
|
+
end
|
153
|
+
rescue StandardError => e
|
154
|
+
handle_callback construct_result(e), mod_ack_req.ack_ids, modack: true
|
103
155
|
end
|
104
156
|
end
|
105
157
|
end
|
106
|
-
|
107
|
-
true
|
108
158
|
end
|
109
159
|
|
110
160
|
def start
|
@@ -115,8 +165,9 @@ module Google
|
|
115
165
|
|
116
166
|
def stop
|
117
167
|
@task.shutdown
|
168
|
+
@retry_thread_pool.shutdown
|
169
|
+
@callback_thread_pool.shutdown
|
118
170
|
flush!
|
119
|
-
|
120
171
|
self
|
121
172
|
end
|
122
173
|
|
@@ -130,6 +181,121 @@ module Google
|
|
130
181
|
|
131
182
|
private
|
132
183
|
|
184
|
+
def handle_failure error, ack_ids, ack_deadline_seconds = nil
|
185
|
+
error_ack_ids = parse_error error, modack: ack_deadline_seconds.nil?
|
186
|
+
unless error_ack_ids.nil?
|
187
|
+
handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS),
|
188
|
+
ack_ids - error_ack_ids,
|
189
|
+
modack: ack_deadline_seconds.nil?
|
190
|
+
ack_ids = error_ack_ids
|
191
|
+
end
|
192
|
+
perform_retry_async ack_ids, ack_deadline_seconds
|
193
|
+
end
|
194
|
+
|
195
|
+
def parse_error error, modack: false
|
196
|
+
metadata = error.error_metadata
|
197
|
+
return if metadata.nil?
|
198
|
+
permanent_failures, temporary_failures = metadata.partition do |_, v|
|
199
|
+
v.include? PERMANENT_FAILURE
|
200
|
+
end.map(&:to_h)
|
201
|
+
unless permanent_failures.empty?
|
202
|
+
handle_callback construct_result(error),
|
203
|
+
permanent_failures.keys.map(&:to_s),
|
204
|
+
modack: modack
|
205
|
+
end
|
206
|
+
temporary_failures.keys.map(&:to_s) unless temporary_failures.empty?
|
207
|
+
end
|
208
|
+
|
209
|
+
def construct_result error
|
210
|
+
case error
|
211
|
+
when Google::Cloud::PermissionDeniedError
|
212
|
+
AcknowledgeResult.new AcknowledgeResult::PERMISSION_DENIED, error
|
213
|
+
when Google::Cloud::FailedPreconditionError
|
214
|
+
AcknowledgeResult.new AcknowledgeResult::FAILED_PRECONDITION, error
|
215
|
+
when Google::Cloud::InvalidArgumentError
|
216
|
+
AcknowledgeResult.new AcknowledgeResult::INVALID_ACK_ID, error
|
217
|
+
else
|
218
|
+
AcknowledgeResult.new AcknowledgeResult::OTHER, error
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def handle_callback result, ack_ids, modack: false
|
223
|
+
ack_ids.each do |ack_id|
|
224
|
+
callback = modack ? @modack_callback_register[ack_id] : @ack_callback_register[ack_id]
|
225
|
+
perform_callback_async result, callback unless callback.nil?
|
226
|
+
end
|
227
|
+
synchronize do
|
228
|
+
if modack
|
229
|
+
@modack_callback_register.delete_if { |ack_id, _| ack_ids.include? ack_id }
|
230
|
+
else
|
231
|
+
@ack_callback_register.delete_if { |ack_id, _| ack_ids.include? ack_id }
|
232
|
+
end
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
def perform_callback_async result, callback
|
237
|
+
return unless retry_thread_pool.running?
|
238
|
+
Concurrent::Promises.future_on(
|
239
|
+
callback_thread_pool, result, callback, &method(:perform_callback_sync)
|
240
|
+
)
|
241
|
+
end
|
242
|
+
|
243
|
+
def perform_callback_sync result, callback
|
244
|
+
begin
|
245
|
+
callback.call result unless stopped?
|
246
|
+
rescue StandardError => e
|
247
|
+
@subscriber.error! e
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
251
|
+
def perform_retry_async ack_ids, ack_deadline_seconds = nil
|
252
|
+
return unless retry_thread_pool.running?
|
253
|
+
Concurrent::Promises.future_on(
|
254
|
+
retry_thread_pool, ack_ids, ack_deadline_seconds, &method(:retry_transient_error)
|
255
|
+
)
|
256
|
+
end
|
257
|
+
|
258
|
+
def retry_transient_error ack_ids, ack_deadline_seconds
|
259
|
+
if ack_deadline_seconds.nil?
|
260
|
+
retry_request ack_ids, ack_deadline_seconds.nil? do |retry_ack_ids|
|
261
|
+
@subscriber.service.acknowledge subscription_name, *retry_ack_ids
|
262
|
+
handle_callback AcknowledgeResult.new AcknowledgeResult::SUCCESS, retry_ack_ids
|
263
|
+
end
|
264
|
+
else
|
265
|
+
retry_request ack_ids, ack_deadline_seconds.nil? do |retry_ack_ids|
|
266
|
+
@subscriber.service.modify_ack_deadline subscription_name, retry_ack_ids, ack_deadline_seconds
|
267
|
+
handle_callback AcknowledgeResult.new AcknowledgeResult::SUCCESS, retry_ack_ids, modack: true
|
268
|
+
end
|
269
|
+
end
|
270
|
+
end
|
271
|
+
|
272
|
+
def retry_request ack_ids, modack
|
273
|
+
begin
|
274
|
+
Retriable.retriable tries: MAX_TRIES,
|
275
|
+
base_interval: BASE_INTERVAL,
|
276
|
+
max_interval: MAX_INTERVAL,
|
277
|
+
multiplier: MULTIPLIER,
|
278
|
+
max_elapsed_time: MAX_RETRY_DURATION,
|
279
|
+
on: EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS do
|
280
|
+
return if ack_ids.nil?
|
281
|
+
begin
|
282
|
+
yield ack_ids
|
283
|
+
rescue Google::Cloud::InvalidArgumentError => e
|
284
|
+
error_ack_ids = parse_error e.error_metadata, modack: modack
|
285
|
+
unless error_ack_ids.nil?
|
286
|
+
handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS),
|
287
|
+
ack_ids - error_ack_ids,
|
288
|
+
modack: modack
|
289
|
+
end
|
290
|
+
ack_ids = error_ack_ids
|
291
|
+
raise e
|
292
|
+
end
|
293
|
+
end
|
294
|
+
rescue StandardError => e
|
295
|
+
handle_callback e, ack_ids, modack: modack
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
133
299
|
def flush_requests!
|
134
300
|
prev_reg =
|
135
301
|
synchronize do
|
@@ -140,13 +306,11 @@ module Google
|
|
140
306
|
end
|
141
307
|
|
142
308
|
groups = prev_reg.each_pair.group_by { |_ack_id, delay| delay }
|
143
|
-
req_hash =
|
309
|
+
req_hash = groups.transform_values { |v| v.map(&:first) }
|
144
310
|
|
145
311
|
requests = { acknowledge: [] }
|
146
312
|
ack_ids = Array(req_hash.delete(:ack)) # ack has no deadline set
|
147
|
-
if ack_ids.any?
|
148
|
-
requests[:acknowledge] = create_acknowledge_requests ack_ids
|
149
|
-
end
|
313
|
+
requests[:acknowledge] = create_acknowledge_requests ack_ids if ack_ids.any?
|
150
314
|
requests[:modify_ack_deadline] =
|
151
315
|
req_hash.map do |mod_deadline, mod_ack_ids|
|
152
316
|
create_modify_ack_deadline_requests mod_deadline, mod_ack_ids
|
@@ -201,8 +365,7 @@ module Google
|
|
201
365
|
end
|
202
366
|
|
203
367
|
def with_threadpool
|
204
|
-
pool = Concurrent::ThreadPoolExecutor.new
|
205
|
-
max_threads: @subscriber.push_threads
|
368
|
+
pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.push_threads
|
206
369
|
|
207
370
|
yield pool
|
208
371
|
|
@@ -213,18 +376,16 @@ module Google
|
|
213
376
|
pool.kill
|
214
377
|
begin
|
215
378
|
raise "Timeout making subscriber API calls"
|
216
|
-
rescue StandardError =>
|
217
|
-
error!
|
379
|
+
rescue StandardError => e
|
380
|
+
error! e
|
218
381
|
end
|
219
382
|
end
|
220
383
|
|
221
384
|
def add_future pool
|
222
385
|
Concurrent::Promises.future_on pool do
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
error! error
|
227
|
-
end
|
386
|
+
yield
|
387
|
+
rescue StandardError => e
|
388
|
+
error! e
|
228
389
|
end
|
229
390
|
end
|
230
391
|
end
|
@@ -41,7 +41,7 @@ module Google
|
|
41
41
|
# subscriber.start
|
42
42
|
#
|
43
43
|
# # Shut down the subscriber when ready to stop receiving messages.
|
44
|
-
# subscriber.stop
|
44
|
+
# subscriber.stop!
|
45
45
|
#
|
46
46
|
# @attr_reader [String] subscription_name The name of the subscription the
|
47
47
|
# messages are pulled from.
|
@@ -50,10 +50,10 @@ module Google
|
|
50
50
|
# @attr_reader [Numeric] deadline The default number of seconds the stream
|
51
51
|
# will hold received messages before modifying the message's ack
|
52
52
|
# deadline. The minimum is 10, the maximum is 600. Default is 60.
|
53
|
+
# @attr_reader [Boolean] message_ordering Whether message ordering has
|
54
|
+
# been enabled.
|
53
55
|
# @attr_reader [Integer] streams The number of concurrent streams to open
|
54
|
-
# to pull messages from the subscription. Default is
|
55
|
-
# @attr_reader [Integer] inventory The number of received messages to be
|
56
|
-
# collected by subscriber. Default is 1,000.
|
56
|
+
# to pull messages from the subscription. Default is 2.
|
57
57
|
# @attr_reader [Integer] callback_threads The number of threads used to
|
58
58
|
# handle the received messages. Default is 8.
|
59
59
|
# @attr_reader [Integer] push_threads The number of threads to handle
|
@@ -64,32 +64,42 @@ module Google
|
|
64
64
|
class Subscriber
|
65
65
|
include MonitorMixin
|
66
66
|
|
67
|
-
attr_reader :subscription_name
|
68
|
-
|
67
|
+
attr_reader :subscription_name
|
68
|
+
attr_reader :callback
|
69
|
+
attr_reader :deadline
|
70
|
+
attr_reader :streams
|
71
|
+
attr_reader :message_ordering
|
72
|
+
attr_reader :callback_threads
|
73
|
+
attr_reader :push_threads
|
69
74
|
|
70
75
|
##
|
71
76
|
# @private Implementation attributes.
|
72
|
-
attr_reader :
|
73
|
-
|
77
|
+
attr_reader :stream_pool, :thread_pool, :buffer, :service
|
78
|
+
|
79
|
+
##
|
80
|
+
# @private Implementation attributes.
|
81
|
+
attr_accessor :exactly_once_delivery_enabled
|
74
82
|
|
75
83
|
##
|
76
84
|
# @private Create an empty {Subscriber} object.
|
77
|
-
def initialize subscription_name, callback, deadline: nil, streams: nil,
|
78
|
-
|
85
|
+
def initialize subscription_name, callback, deadline: nil, message_ordering: nil, streams: nil, inventory: nil,
|
86
|
+
threads: {}, service: nil
|
87
|
+
super() # to init MonitorMixin
|
88
|
+
|
79
89
|
@callback = callback
|
80
90
|
@error_callbacks = []
|
81
91
|
@subscription_name = subscription_name
|
82
92
|
@deadline = deadline || 60
|
83
|
-
@streams = streams ||
|
84
|
-
|
85
|
-
@
|
86
|
-
@
|
93
|
+
@streams = streams || 2
|
94
|
+
coerce_inventory inventory
|
95
|
+
@message_ordering = message_ordering
|
96
|
+
@callback_threads = Integer(threads[:callback] || 8)
|
97
|
+
@push_threads = Integer(threads[:push] || 4)
|
98
|
+
@exactly_once_delivery_enabled = nil
|
87
99
|
|
88
|
-
@stream_inventory = @inventory.fdiv(@streams).ceil
|
89
100
|
@service = service
|
90
101
|
|
91
|
-
@started = nil
|
92
|
-
@stopped = nil
|
102
|
+
@started = @stopped = nil
|
93
103
|
|
94
104
|
stream_pool = Array.new @streams do
|
95
105
|
Thread.new { Stream.new self }
|
@@ -97,8 +107,6 @@ module Google
|
|
97
107
|
@stream_pool = stream_pool.map(&:value)
|
98
108
|
|
99
109
|
@buffer = TimedUnaryBuffer.new self
|
100
|
-
|
101
|
-
super() # to init MonitorMixin
|
102
110
|
end
|
103
111
|
|
104
112
|
##
|
@@ -106,6 +114,7 @@ module Google
|
|
106
114
|
# received messages.
|
107
115
|
#
|
108
116
|
# @return [Subscriber] returns self so calls can be chained.
|
117
|
+
#
|
109
118
|
def start
|
110
119
|
start_pool = synchronize do
|
111
120
|
@started = true
|
@@ -124,27 +133,21 @@ module Google
|
|
124
133
|
|
125
134
|
##
|
126
135
|
# Immediately stops the subscriber. No new messages will be pulled from
|
127
|
-
# the subscription.
|
128
|
-
#
|
129
|
-
#
|
130
|
-
#
|
131
|
-
# received messages have been processed or released.
|
136
|
+
# the subscription. Use {#wait!} to block until all received messages have
|
137
|
+
# been processed or released: All actions taken on received messages that
|
138
|
+
# have not yet been sent to the API will be sent to the API. All received
|
139
|
+
# but unprocessed messages will be released back to the API and redelivered.
|
132
140
|
#
|
133
141
|
# @return [Subscriber] returns self so calls can be chained.
|
142
|
+
#
|
134
143
|
def stop
|
135
|
-
|
144
|
+
synchronize do
|
136
145
|
@started = false
|
137
146
|
@stopped = true
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
end
|
147
|
+
@stream_pool.map(&:stop)
|
148
|
+
wait_stop_buffer_thread!
|
149
|
+
self
|
142
150
|
end
|
143
|
-
stop_pool.map(&:join)
|
144
|
-
# Stop the buffer after the streams are all stopped
|
145
|
-
synchronize { @buffer.stop }
|
146
|
-
|
147
|
-
self
|
148
151
|
end
|
149
152
|
|
150
153
|
##
|
@@ -160,14 +163,10 @@ module Google
|
|
160
163
|
# subscriber is fully stopped. Default will block indefinitely.
|
161
164
|
#
|
162
165
|
# @return [Subscriber] returns self so calls can be chained.
|
166
|
+
#
|
163
167
|
def wait! timeout = nil
|
164
|
-
|
165
|
-
|
166
|
-
Thread.new { stream.wait! timeout }
|
167
|
-
end
|
168
|
-
end
|
169
|
-
wait_pool.map(&:join)
|
170
|
-
|
168
|
+
wait_stop_buffer_thread!
|
169
|
+
@wait_stop_buffer_thread.join timeout
|
171
170
|
self
|
172
171
|
end
|
173
172
|
|
@@ -192,6 +191,7 @@ module Google
|
|
192
191
|
# Whether the subscriber has been started.
|
193
192
|
#
|
194
193
|
# @return [boolean] `true` when started, `false` otherwise.
|
194
|
+
#
|
195
195
|
def started?
|
196
196
|
synchronize { @started }
|
197
197
|
end
|
@@ -200,6 +200,7 @@ module Google
|
|
200
200
|
# Whether the subscriber has been stopped.
|
201
201
|
#
|
202
202
|
# @return [boolean] `true` when stopped, `false` otherwise.
|
203
|
+
#
|
203
204
|
def stopped?
|
204
205
|
synchronize { @stopped }
|
205
206
|
end
|
@@ -237,7 +238,7 @@ module Google
|
|
237
238
|
# subscriber.start
|
238
239
|
#
|
239
240
|
# # Shut down the subscriber when ready to stop receiving messages.
|
240
|
-
# subscriber.stop
|
241
|
+
# subscriber.stop!
|
241
242
|
#
|
242
243
|
def on_error &block
|
243
244
|
synchronize do
|
@@ -273,12 +274,91 @@ module Google
|
|
273
274
|
# subscriber.last_error #=> nil
|
274
275
|
#
|
275
276
|
# # Shut down the subscriber when ready to stop receiving messages.
|
276
|
-
# subscriber.stop
|
277
|
+
# subscriber.stop!
|
277
278
|
#
|
278
279
|
def last_error
|
279
280
|
synchronize { @last_error }
|
280
281
|
end
|
281
282
|
|
283
|
+
##
|
284
|
+
# The number of received messages to be collected by subscriber. Default is 1,000.
|
285
|
+
#
|
286
|
+
# @return [Integer] The maximum number of messages.
|
287
|
+
#
|
288
|
+
def max_outstanding_messages
|
289
|
+
@inventory[:max_outstanding_messages]
|
290
|
+
end
|
291
|
+
# @deprecated Use {#max_outstanding_messages}.
|
292
|
+
alias inventory_limit max_outstanding_messages
|
293
|
+
# @deprecated Use {#max_outstanding_messages}.
|
294
|
+
alias inventory max_outstanding_messages
|
295
|
+
|
296
|
+
##
|
297
|
+
# The total byte size of received messages to be collected by subscriber. Default is 100,000,000 (100MB).
|
298
|
+
#
|
299
|
+
# @return [Integer] The maximum number of bytes.
|
300
|
+
#
|
301
|
+
def max_outstanding_bytes
|
302
|
+
@inventory[:max_outstanding_bytes]
|
303
|
+
end
|
304
|
+
# @deprecated Use {#max_outstanding_bytes}.
|
305
|
+
alias inventory_bytesize max_outstanding_bytes
|
306
|
+
|
307
|
+
##
|
308
|
+
# Whether to enforce flow control at the client side only or to enforce it at both the client and
|
309
|
+
# the server. For more details about flow control see https://cloud.google.com/pubsub/docs/pull#config.
|
310
|
+
#
|
311
|
+
# @return [Boolean] `true` when only client side flow control is enforced, `false` when both client and
|
312
|
+
# server side flow control are enforced.
|
313
|
+
#
|
314
|
+
def use_legacy_flow_control?
|
315
|
+
@inventory[:use_legacy_flow_control]
|
316
|
+
end
|
317
|
+
|
318
|
+
##
|
319
|
+
# The number of seconds that received messages can be held awaiting processing. Default is 3,600 (1 hour).
|
320
|
+
#
|
321
|
+
# @return [Integer] The maximum number of seconds.
|
322
|
+
#
|
323
|
+
def max_total_lease_duration
|
324
|
+
@inventory[:max_total_lease_duration]
|
325
|
+
end
|
326
|
+
# @deprecated Use {#max_total_lease_duration}.
|
327
|
+
alias inventory_extension max_total_lease_duration
|
328
|
+
|
329
|
+
##
|
330
|
+
# The maximum amount of time in seconds for a single lease extension attempt. Bounds the delay before a message
|
331
|
+
# redelivery if the subscriber fails to extend the deadline. Default is 0 (disabled).
|
332
|
+
#
|
333
|
+
# @return [Integer] The maximum number of seconds.
|
334
|
+
#
|
335
|
+
def max_duration_per_lease_extension
|
336
|
+
@inventory[:max_duration_per_lease_extension]
|
337
|
+
end
|
338
|
+
|
339
|
+
##
|
340
|
+
# The minimum amount of time in seconds for a single lease extension attempt. Bounds the delay before a message
|
341
|
+
# redelivery if the subscriber fails to extend the deadline. Default is 0 (disabled).
|
342
|
+
#
|
343
|
+
# @return [Integer] The minimum number of seconds.
|
344
|
+
#
|
345
|
+
def min_duration_per_lease_extension
|
346
|
+
@inventory[:min_duration_per_lease_extension]
|
347
|
+
end
|
348
|
+
|
349
|
+
##
|
350
|
+
# @private
|
351
|
+
def stream_inventory
|
352
|
+
{
|
353
|
+
limit: @inventory[:max_outstanding_messages].fdiv(@streams).ceil,
|
354
|
+
bytesize: @inventory[:max_outstanding_bytes].fdiv(@streams).ceil,
|
355
|
+
extension: @inventory[:max_total_lease_duration],
|
356
|
+
max_duration_per_lease_extension: @inventory[:max_duration_per_lease_extension],
|
357
|
+
min_duration_per_lease_extension: @inventory[:min_duration_per_lease_extension],
|
358
|
+
use_legacy_flow_control: @inventory[:use_legacy_flow_control]
|
359
|
+
}
|
360
|
+
end
|
361
|
+
|
282
362
|
# @private returns error object from the stream thread.
|
283
363
|
def error! error
|
284
364
|
error_callbacks = synchronize do
|
@@ -292,8 +372,7 @@ module Google
|
|
292
372
|
##
|
293
373
|
# @private
|
294
374
|
def to_s
|
295
|
-
"(subscription: #{subscription_name}, "
|
296
|
-
"streams: [#{stream_pool.map(&:to_s).join(', ')}])"
|
375
|
+
"(subscription: #{subscription_name}, streams: [#{stream_pool.map(&:to_s).join(', ')}])"
|
297
376
|
end
|
298
377
|
|
299
378
|
##
|
@@ -304,6 +383,37 @@ module Google
|
|
304
383
|
|
305
384
|
protected
|
306
385
|
|
386
|
+
##
|
387
|
+
# Starts a new thread to call wait! (blocking) on each Stream and then stop the TimedUnaryBuffer.
|
388
|
+
def wait_stop_buffer_thread!
|
389
|
+
synchronize do
|
390
|
+
@wait_stop_buffer_thread ||= Thread.new do
|
391
|
+
@stream_pool.map(&:wait!)
|
392
|
+
# Shutdown the buffer TimerTask (and flush the buffer) after the streams are all stopped.
|
393
|
+
@buffer.stop
|
394
|
+
end
|
395
|
+
end
|
396
|
+
end
|
397
|
+
|
398
|
+
def coerce_inventory inventory
|
399
|
+
@inventory = inventory
|
400
|
+
if @inventory.is_a? Hash
|
401
|
+
@inventory = @inventory.dup
|
402
|
+
# Support deprecated field names
|
403
|
+
@inventory[:max_outstanding_messages] ||= @inventory.delete :limit
|
404
|
+
@inventory[:max_outstanding_bytes] ||= @inventory.delete :bytesize
|
405
|
+
@inventory[:max_total_lease_duration] ||= @inventory.delete :extension
|
406
|
+
else
|
407
|
+
@inventory = { max_outstanding_messages: @inventory }
|
408
|
+
end
|
409
|
+
@inventory[:max_outstanding_messages] = Integer(@inventory[:max_outstanding_messages] || 1000)
|
410
|
+
@inventory[:max_outstanding_bytes] = Integer(@inventory[:max_outstanding_bytes] || 100_000_000)
|
411
|
+
@inventory[:max_total_lease_duration] = Integer(@inventory[:max_total_lease_duration] || 3600)
|
412
|
+
@inventory[:max_duration_per_lease_extension] = Integer(@inventory[:max_duration_per_lease_extension] || 0)
|
413
|
+
@inventory[:min_duration_per_lease_extension] = Integer(@inventory[:min_duration_per_lease_extension] || 0)
|
414
|
+
@inventory[:use_legacy_flow_control] = @inventory[:use_legacy_flow_control] || false
|
415
|
+
end
|
416
|
+
|
307
417
|
def default_error_callbacks
|
308
418
|
# This is memoized to reduce calls to the configuration.
|
309
419
|
@default_error_callbacks ||= begin
|
@@ -130,17 +130,15 @@ module Google
|
|
130
130
|
# puts subscription.name
|
131
131
|
# end
|
132
132
|
#
|
133
|
-
def all request_limit: nil
|
133
|
+
def all request_limit: nil, &block
|
134
134
|
request_limit = request_limit.to_i if request_limit
|
135
|
-
unless block_given?
|
136
|
-
return enum_for :all, request_limit: request_limit
|
137
|
-
end
|
135
|
+
return enum_for :all, request_limit: request_limit unless block_given?
|
138
136
|
results = self
|
139
137
|
loop do
|
140
|
-
results.each
|
138
|
+
results.each(&block)
|
141
139
|
if request_limit
|
142
140
|
request_limit -= 1
|
143
|
-
break if request_limit
|
141
|
+
break if request_limit.negative?
|
144
142
|
end
|
145
143
|
break unless results.next?
|
146
144
|
results = results.next
|