google-cloud-pubsub 2.10.0 → 2.12.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +23 -0
- data/lib/google/cloud/pubsub/acknowledge_result.rb +79 -0
- data/lib/google/cloud/pubsub/project.rb +1 -1
- data/lib/google/cloud/pubsub/received_message.rb +87 -6
- data/lib/google/cloud/pubsub/service.rb +30 -24
- data/lib/google/cloud/pubsub/subscriber/stream.rb +5 -4
- data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb +176 -9
- data/lib/google/cloud/pubsub/subscriber.rb +5 -0
- data/lib/google/cloud/pubsub/topic.rb +33 -46
- data/lib/google/cloud/pubsub/version.rb +1 -1
- metadata +21 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: afdce505d87a251a0408c830e80728cf68506cf5631013cfcdae0bc18dba549b
|
4
|
+
data.tar.gz: c46a1e91681598c21d1b4014b6fb0aadb7f8575c9c08e2c9a13ac8b91de87ed6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6a31118080d4ad3a6a03b1911f6ec192dfe51021140bea354568c128856bd86cfbea79fd75d6fc6457a68e7266f018f9ef1e23868ee074576539f71916f8bc57
|
7
|
+
data.tar.gz: 4b9ff30c34dd3e08456dce04434e7e9da0552048dc86980d3b82f0af1114377601c7a96d71e1738973df0a506302dc5a7919ac900616b7b2fca58951891e5dfc
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,28 @@
|
|
1
1
|
# Release History
|
2
2
|
|
3
|
+
### 2.12.1 (2022-08-21)
|
4
|
+
|
5
|
+
#### Bug Fixes
|
6
|
+
|
7
|
+
* update non EOS ack to return Success always ([#19023](https://github.com/googleapis/google-cloud-ruby/issues/19023))
|
8
|
+
|
9
|
+
### 2.12.0 (2022-08-09)
|
10
|
+
|
11
|
+
#### Features
|
12
|
+
|
13
|
+
* bump the minimum required version of pubsub v1 ([#18983](https://github.com/googleapis/google-cloud-ruby/issues/18983))
|
14
|
+
#### Bug Fixes
|
15
|
+
|
16
|
+
* honour async options of topic ([#18953](https://github.com/googleapis/google-cloud-ruby/issues/18953))
|
17
|
+
|
18
|
+
### 2.11.0 (2022-08-01)
|
19
|
+
|
20
|
+
#### Features
|
21
|
+
|
22
|
+
* create exactly once delivery enabled subscription ([#18824](https://github.com/googleapis/google-cloud-ruby/issues/18824))
|
23
|
+
* Let user register callback and get acknowledgement result ([#18702](https://github.com/googleapis/google-cloud-ruby/issues/18702))
|
24
|
+
* retry transient failures in ack/modack in timed unary buffer ([#18395](https://github.com/googleapis/google-cloud-ruby/issues/18395))
|
25
|
+
|
3
26
|
### 2.10.0 (2022-06-14)
|
4
27
|
|
5
28
|
#### Features
|
@@ -0,0 +1,79 @@
|
|
1
|
+
# Copyright 2022 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
module Google
|
17
|
+
module Cloud
|
18
|
+
module PubSub
|
19
|
+
##
|
20
|
+
# The result of a ack/nack/modack on messages.
|
21
|
+
#
|
22
|
+
# When the operation was successful the result will be marked
|
23
|
+
# {#succeeded?}. Otherwise, the result will be marked {#failed?} and the
|
24
|
+
# error raised will be availabe on {#error}.
|
25
|
+
#
|
26
|
+
class AcknowledgeResult
|
27
|
+
##
|
28
|
+
# The constants below represents the status of ack/modack operations.
|
29
|
+
# Indicates successful ack/modack
|
30
|
+
SUCCESS = 1
|
31
|
+
|
32
|
+
##
|
33
|
+
# Indicates occurence of permenant permission denied error
|
34
|
+
PERMISSION_DENIED = 2
|
35
|
+
|
36
|
+
##
|
37
|
+
# Indicates occurence of permenant failed precondition error
|
38
|
+
FAILED_PRECONDITION = 3
|
39
|
+
|
40
|
+
##
|
41
|
+
# Indicates occurence of permenant permission denied error
|
42
|
+
INVALID_ACK_ID = 4
|
43
|
+
|
44
|
+
##
|
45
|
+
# Indicates occurence of permenant uncatogorised error
|
46
|
+
OTHER = 5
|
47
|
+
|
48
|
+
##
|
49
|
+
# @return [Google::Cloud::Error] Error object of ack/modack operation
|
50
|
+
attr_reader :error
|
51
|
+
|
52
|
+
##
|
53
|
+
# @return [Numeric] Status of the ack/modack operation.
|
54
|
+
attr_reader :status
|
55
|
+
|
56
|
+
##
|
57
|
+
# @private Create an PublishResult object.
|
58
|
+
def initialize status, error = nil
|
59
|
+
@error = error
|
60
|
+
@status = status
|
61
|
+
end
|
62
|
+
|
63
|
+
##
|
64
|
+
# @return [Boolean] Whether the operation was successful.
|
65
|
+
def succeeded?
|
66
|
+
@status == SUCCESS
|
67
|
+
end
|
68
|
+
|
69
|
+
##
|
70
|
+
# @return [Boolean] Whether the operation failed.
|
71
|
+
def failed?
|
72
|
+
!succeeded?
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
Pubsub = PubSub unless const_defined? :Pubsub
|
78
|
+
end
|
79
|
+
end
|
@@ -160,7 +160,7 @@ module Google
|
|
160
160
|
#
|
161
161
|
def topic topic_name, project: nil, skip_lookup: nil, async: nil
|
162
162
|
ensure_service!
|
163
|
-
options = { project: project }
|
163
|
+
options = { project: project, async: async }
|
164
164
|
return Topic.from_name topic_name, service, options if skip_lookup
|
165
165
|
grpc = service.get_topic topic_name, options
|
166
166
|
Topic.from_grpc grpc, service, async: async
|
@@ -161,6 +161,29 @@ module Google
|
|
161
161
|
##
|
162
162
|
# Acknowledges receipt of the message.
|
163
163
|
#
|
164
|
+
# @yield [callback] The block to be called when reject operation is done.
|
165
|
+
# @yieldparam [Google::Cloud::PubSub::AcknowledgeResult] Result object that contains the status and error.
|
166
|
+
#
|
167
|
+
# @example
|
168
|
+
# require "google/cloud/pubsub"
|
169
|
+
#
|
170
|
+
# pubsub = Google::Cloud::PubSub.new
|
171
|
+
#
|
172
|
+
# sub = pubsub.subscription "my-topic-sub"
|
173
|
+
# subscriber = sub.listen do |received_message|
|
174
|
+
# puts received_message.message.data
|
175
|
+
#
|
176
|
+
# received_message.acknowledge! do |result|
|
177
|
+
# puts result.status
|
178
|
+
# end
|
179
|
+
# end
|
180
|
+
#
|
181
|
+
# # Start background threads that will call block passed to listen.
|
182
|
+
# subscriber.start
|
183
|
+
#
|
184
|
+
# # Shut down the subscriber when ready to stop receiving messages.
|
185
|
+
# subscriber.stop!
|
186
|
+
#
|
164
187
|
# @example
|
165
188
|
# require "google/cloud/pubsub"
|
166
189
|
#
|
@@ -179,9 +202,14 @@ module Google
|
|
179
202
|
# # Shut down the subscriber when ready to stop receiving messages.
|
180
203
|
# subscriber.stop!
|
181
204
|
#
|
182
|
-
def acknowledge!
|
205
|
+
def acknowledge! &block
|
183
206
|
ensure_subscription!
|
184
|
-
subscription.
|
207
|
+
if subscription.respond_to?(:exactly_once_delivery_enabled) && subscription.exactly_once_delivery_enabled
|
208
|
+
subscription.acknowledge ack_id, &block
|
209
|
+
else
|
210
|
+
subscription.acknowledge ack_id
|
211
|
+
yield AcknowledgeResult.new(AcknowledgeResult::SUCCESS) if block_given?
|
212
|
+
end
|
185
213
|
end
|
186
214
|
alias ack! acknowledge!
|
187
215
|
|
@@ -197,6 +225,30 @@ module Google
|
|
197
225
|
# seconds after the call is made. Specifying `0` may immediately make
|
198
226
|
# the message available for another pull request.
|
199
227
|
#
|
228
|
+
# @yield [callback] The block to be called when reject operation is done.
|
229
|
+
# @yieldparam [Google::Cloud::PubSub::AcknowledgeResult] Result object that contains the status and error.
|
230
|
+
#
|
231
|
+
# @example
|
232
|
+
# require "google/cloud/pubsub"
|
233
|
+
#
|
234
|
+
# pubsub = Google::Cloud::PubSub.new
|
235
|
+
#
|
236
|
+
# sub = pubsub.subscription "my-topic-sub"
|
237
|
+
# subscriber = sub.listen do |received_message|
|
238
|
+
# puts received_message.message.data
|
239
|
+
#
|
240
|
+
# # Delay for 2 minutes
|
241
|
+
# received_message.modify_ack_deadline! 120 do |result|
|
242
|
+
# puts result.status
|
243
|
+
# end
|
244
|
+
# end
|
245
|
+
#
|
246
|
+
# # Start background threads that will call block passed to listen.
|
247
|
+
# subscriber.start
|
248
|
+
#
|
249
|
+
# # Shut down the subscriber when ready to stop receiving messages.
|
250
|
+
# subscriber.stop!
|
251
|
+
#
|
200
252
|
# @example
|
201
253
|
# require "google/cloud/pubsub"
|
202
254
|
#
|
@@ -216,9 +268,14 @@ module Google
|
|
216
268
|
# # Shut down the subscriber when ready to stop receiving messages.
|
217
269
|
# subscriber.stop!
|
218
270
|
#
|
219
|
-
def modify_ack_deadline! new_deadline
|
271
|
+
def modify_ack_deadline! new_deadline, &block
|
220
272
|
ensure_subscription!
|
221
|
-
subscription.
|
273
|
+
if subscription.respond_to?(:exactly_once_delivery_enabled) && subscription.exactly_once_delivery_enabled
|
274
|
+
subscription.modify_ack_deadline new_deadline, ack_id, &block
|
275
|
+
else
|
276
|
+
subscription.modify_ack_deadline new_deadline, ack_id
|
277
|
+
yield AcknowledgeResult.new(AcknowledgeResult::SUCCESS) if block_given?
|
278
|
+
end
|
222
279
|
end
|
223
280
|
|
224
281
|
##
|
@@ -227,6 +284,30 @@ module Google
|
|
227
284
|
#
|
228
285
|
# This will make the message available for redelivery.
|
229
286
|
#
|
287
|
+
# @yield [callback] The block to be called when reject operation is done.
|
288
|
+
# @yieldparam [Google::Cloud::PubSub::AcknowledgeResult] Result object that contains the status and error.
|
289
|
+
#
|
290
|
+
# @example
|
291
|
+
# require "google/cloud/pubsub"
|
292
|
+
#
|
293
|
+
# pubsub = Google::Cloud::PubSub.new
|
294
|
+
#
|
295
|
+
# sub = pubsub.subscription "my-topic-sub"
|
296
|
+
# subscriber = sub.listen do |received_message|
|
297
|
+
# puts received_message.message.data
|
298
|
+
#
|
299
|
+
# # Release message back to the API.
|
300
|
+
# received_message.reject! do |result|
|
301
|
+
# puts result.status
|
302
|
+
# end
|
303
|
+
# end
|
304
|
+
#
|
305
|
+
# # Start background threads that will call block passed to listen.
|
306
|
+
# subscriber.start
|
307
|
+
#
|
308
|
+
# # Shut down the subscriber when ready to stop receiving messages.
|
309
|
+
# subscriber.stop!
|
310
|
+
#
|
230
311
|
# @example
|
231
312
|
# require "google/cloud/pubsub"
|
232
313
|
#
|
@@ -246,8 +327,8 @@ module Google
|
|
246
327
|
# # Shut down the subscriber when ready to stop receiving messages.
|
247
328
|
# subscriber.stop!
|
248
329
|
#
|
249
|
-
def reject!
|
250
|
-
modify_ack_deadline! 0
|
330
|
+
def reject! &block
|
331
|
+
modify_ack_deadline! 0, &block
|
251
332
|
end
|
252
333
|
alias nack! reject!
|
253
334
|
alias ignore! reject!
|
@@ -204,18 +204,8 @@ module Google
|
|
204
204
|
##
|
205
205
|
# Creates a subscription on a given topic for a given subscriber.
|
206
206
|
def create_subscription topic, subscription_name, options = {}
|
207
|
-
|
208
|
-
|
209
|
-
topic: topic_path(topic),
|
210
|
-
push_config: options[:push_config],
|
211
|
-
ack_deadline_seconds: options[:deadline],
|
212
|
-
retain_acked_messages: options[:retain_acked],
|
213
|
-
message_retention_duration: Convert.number_to_duration(options[:retention]),
|
214
|
-
labels: options[:labels],
|
215
|
-
enable_message_ordering: options[:message_ordering],
|
216
|
-
filter: options[:filter],
|
217
|
-
dead_letter_policy: dead_letter_policy(options),
|
218
|
-
retry_policy: options[:retry_policy]
|
207
|
+
updated_option = construct_create_subscription_options topic, subscription_name, options
|
208
|
+
subscriber.create_subscription(**updated_option)
|
219
209
|
end
|
220
210
|
|
221
211
|
def update_subscription subscription_obj, *fields
|
@@ -255,11 +245,7 @@ module Google
|
|
255
245
|
##
|
256
246
|
# Acknowledges receipt of a message.
|
257
247
|
def acknowledge subscription, *ack_ids
|
258
|
-
|
259
|
-
subscriber.acknowledge subscription: subscription_path(subscription), ack_ids: ack_ids
|
260
|
-
rescue StandardError => e
|
261
|
-
raise e unless e.cause.is_a? GRPC::BadStatus
|
262
|
-
end
|
248
|
+
subscriber.acknowledge subscription: subscription_path(subscription), ack_ids: ack_ids
|
263
249
|
end
|
264
250
|
|
265
251
|
##
|
@@ -279,13 +265,9 @@ module Google
|
|
279
265
|
##
|
280
266
|
# Modifies the ack deadline for a specific message.
|
281
267
|
def modify_ack_deadline subscription, ids, deadline
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
ack_deadline_seconds: deadline
|
286
|
-
rescue StandardError => e
|
287
|
-
raise e unless e.cause.is_a? GRPC::BadStatus
|
288
|
-
end
|
268
|
+
subscriber.modify_ack_deadline subscription: subscription_path(subscription),
|
269
|
+
ack_ids: Array(ids),
|
270
|
+
ack_deadline_seconds: deadline
|
289
271
|
end
|
290
272
|
|
291
273
|
##
|
@@ -496,6 +478,30 @@ module Google
|
|
496
478
|
end
|
497
479
|
policy
|
498
480
|
end
|
481
|
+
|
482
|
+
private
|
483
|
+
|
484
|
+
def construct_create_subscription_options topic, subscription_name, options
|
485
|
+
excess_options = [:deadline,
|
486
|
+
:retention,
|
487
|
+
:retain_acked,
|
488
|
+
:message_ordering,
|
489
|
+
:endpoint,
|
490
|
+
:dead_letter_topic_name,
|
491
|
+
:dead_letter_max_delivery_attempts,
|
492
|
+
:dead_letter_topic]
|
493
|
+
|
494
|
+
new_options = options.filter { |k, v| !v.nil? && !excess_options.include?(k) }
|
495
|
+
new_options[:name] = subscription_path subscription_name, options
|
496
|
+
new_options[:topic] = topic_path topic
|
497
|
+
new_options[:message_retention_duration] = Convert.number_to_duration options[:retention]
|
498
|
+
new_options[:dead_letter_policy] = dead_letter_policy options
|
499
|
+
new_options[:ack_deadline_seconds] = options[:deadline]
|
500
|
+
new_options[:retain_acked_messages] = options[:retain_acked]
|
501
|
+
new_options[:enable_message_ordering] = options[:message_ordering]
|
502
|
+
|
503
|
+
new_options.compact
|
504
|
+
end
|
499
505
|
end
|
500
506
|
end
|
501
507
|
|
@@ -134,13 +134,13 @@ module Google
|
|
134
134
|
|
135
135
|
##
|
136
136
|
# @private
|
137
|
-
def acknowledge *messages
|
137
|
+
def acknowledge *messages, &callback
|
138
138
|
ack_ids = coerce_ack_ids messages
|
139
139
|
return true if ack_ids.empty?
|
140
140
|
|
141
141
|
synchronize do
|
142
142
|
@inventory.remove ack_ids
|
143
|
-
@subscriber.buffer.acknowledge ack_ids
|
143
|
+
@subscriber.buffer.acknowledge ack_ids, callback
|
144
144
|
end
|
145
145
|
|
146
146
|
true
|
@@ -148,13 +148,13 @@ module Google
|
|
148
148
|
|
149
149
|
##
|
150
150
|
# @private
|
151
|
-
def modify_ack_deadline deadline, *messages
|
151
|
+
def modify_ack_deadline deadline, *messages, &callback
|
152
152
|
mod_ack_ids = coerce_ack_ids messages
|
153
153
|
return true if mod_ack_ids.empty?
|
154
154
|
|
155
155
|
synchronize do
|
156
156
|
@inventory.remove mod_ack_ids
|
157
|
-
@subscriber.buffer.modify_ack_deadline deadline, mod_ack_ids
|
157
|
+
@subscriber.buffer.modify_ack_deadline deadline, mod_ack_ids, callback
|
158
158
|
end
|
159
159
|
|
160
160
|
true
|
@@ -253,6 +253,7 @@ module Google
|
|
253
253
|
synchronize do
|
254
254
|
update_min_duration_per_lease_extension new_exactly_once_delivery_enabled
|
255
255
|
@exactly_once_delivery_enabled = new_exactly_once_delivery_enabled unless new_exactly_once_delivery_enabled.nil?
|
256
|
+
@subscriber.exactly_once_delivery_enabled = @exactly_once_delivery_enabled
|
256
257
|
|
257
258
|
# Create receipt of received messages reception
|
258
259
|
@subscriber.buffer.modify_ack_deadline @subscriber.deadline, response.received_messages.map(&:ack_id)
|
@@ -14,7 +14,10 @@
|
|
14
14
|
|
15
15
|
|
16
16
|
require "concurrent"
|
17
|
+
require "google/cloud/errors"
|
18
|
+
require "google/cloud/pubsub/acknowledge_result"
|
17
19
|
require "monitor"
|
20
|
+
require "retriable"
|
18
21
|
|
19
22
|
module Google
|
20
23
|
module Cloud
|
@@ -27,6 +30,21 @@ module Google
|
|
27
30
|
|
28
31
|
attr_reader :max_bytes
|
29
32
|
attr_reader :interval
|
33
|
+
attr_reader :retry_thread_pool
|
34
|
+
attr_reader :callback_thread_pool
|
35
|
+
|
36
|
+
PERMANENT_FAILURE = "PERMANENT_FAILURE".freeze
|
37
|
+
# Google::Cloud::Unavailable error is already retried at gapic level
|
38
|
+
EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS = [Google::Cloud::CanceledError,
|
39
|
+
Google::Cloud::DeadlineExceededError,
|
40
|
+
Google::Cloud::InternalError,
|
41
|
+
Google::Cloud::ResourceExhaustedError,
|
42
|
+
Google::Cloud::InvalidArgumentError].freeze
|
43
|
+
MAX_RETRY_DURATION = 600 # 600s since the server allows ack/modacks for 10 mins max
|
44
|
+
MAX_TRIES = 15
|
45
|
+
BASE_INTERVAL = 1
|
46
|
+
MAX_INTERVAL = 64
|
47
|
+
MULTIPLIER = 2
|
30
48
|
|
31
49
|
def initialize subscriber, max_bytes: 500_000, interval: 1.0
|
32
50
|
super() # to init MonitorMixin
|
@@ -40,30 +58,37 @@ module Google
|
|
40
58
|
# entry.
|
41
59
|
@register = {}
|
42
60
|
|
61
|
+
@ack_callback_register = {}
|
62
|
+
@modack_callback_register = {}
|
63
|
+
|
64
|
+
@retry_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
|
65
|
+
@callback_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
|
43
66
|
@task = Concurrent::TimerTask.new execution_interval: interval do
|
44
67
|
flush!
|
45
68
|
end
|
46
69
|
end
|
47
70
|
|
48
|
-
def acknowledge ack_ids
|
71
|
+
def acknowledge ack_ids, callback = nil
|
49
72
|
return if ack_ids.empty?
|
50
73
|
|
51
74
|
synchronize do
|
52
75
|
ack_ids.each do |ack_id|
|
53
76
|
# ack has no deadline set, use :ack indicate it is an ack
|
54
77
|
@register[ack_id] = :ack
|
78
|
+
@ack_callback_register[ack_id] = callback unless callback.nil?
|
55
79
|
end
|
56
80
|
end
|
57
81
|
|
58
82
|
true
|
59
83
|
end
|
60
84
|
|
61
|
-
def modify_ack_deadline deadline, ack_ids
|
85
|
+
def modify_ack_deadline deadline, ack_ids, callback = nil
|
62
86
|
return if ack_ids.empty?
|
63
87
|
|
64
88
|
synchronize do
|
65
89
|
ack_ids.each do |ack_id|
|
66
90
|
@register[ack_id] = deadline
|
91
|
+
@modack_callback_register[ack_id] = callback unless callback.nil?
|
67
92
|
end
|
68
93
|
end
|
69
94
|
|
@@ -90,20 +115,46 @@ module Google
|
|
90
115
|
|
91
116
|
# Perform the RCP calls concurrently
|
92
117
|
with_threadpool do |pool|
|
93
|
-
requests
|
94
|
-
|
118
|
+
make_acknowledge_request requests, pool
|
119
|
+
make_modack_request requests, pool
|
120
|
+
end
|
121
|
+
|
122
|
+
true
|
123
|
+
end
|
124
|
+
|
125
|
+
def make_acknowledge_request requests, pool
|
126
|
+
requests[:acknowledge].each do |ack_req|
|
127
|
+
add_future pool do
|
128
|
+
begin
|
95
129
|
@subscriber.service.acknowledge ack_req.subscription, *ack_req.ack_ids
|
130
|
+
handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS), ack_req.ack_ids
|
131
|
+
rescue *EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS => e
|
132
|
+
handle_failure e, ack_req.ack_ids if @subscriber.exactly_once_delivery_enabled
|
133
|
+
rescue StandardError => e
|
134
|
+
handle_callback construct_result(e), ack_req.ack_ids
|
96
135
|
end
|
97
136
|
end
|
98
|
-
|
99
|
-
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
def make_modack_request requests, pool
|
141
|
+
requests[:modify_ack_deadline].each do |mod_ack_req|
|
142
|
+
add_future pool do
|
143
|
+
begin
|
100
144
|
@subscriber.service.modify_ack_deadline mod_ack_req.subscription, mod_ack_req.ack_ids,
|
101
145
|
mod_ack_req.ack_deadline_seconds
|
146
|
+
handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS),
|
147
|
+
mod_ack_req.ack_ids,
|
148
|
+
modack: true
|
149
|
+
rescue *EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS => e
|
150
|
+
if @subscriber.exactly_once_delivery_enabled
|
151
|
+
handle_failure e, mod_ack_req.ack_ids, mod_ack_req.ack_deadline_seconds
|
152
|
+
end
|
153
|
+
rescue StandardError => e
|
154
|
+
handle_callback construct_result(e), mod_ack_req.ack_ids, modack: true
|
102
155
|
end
|
103
156
|
end
|
104
157
|
end
|
105
|
-
|
106
|
-
true
|
107
158
|
end
|
108
159
|
|
109
160
|
def start
|
@@ -114,8 +165,9 @@ module Google
|
|
114
165
|
|
115
166
|
def stop
|
116
167
|
@task.shutdown
|
168
|
+
@retry_thread_pool.shutdown
|
169
|
+
@callback_thread_pool.shutdown
|
117
170
|
flush!
|
118
|
-
|
119
171
|
self
|
120
172
|
end
|
121
173
|
|
@@ -129,6 +181,121 @@ module Google
|
|
129
181
|
|
130
182
|
private
|
131
183
|
|
184
|
+
def handle_failure error, ack_ids, ack_deadline_seconds = nil
|
185
|
+
error_ack_ids = parse_error error, modack: ack_deadline_seconds.nil?
|
186
|
+
unless error_ack_ids.nil?
|
187
|
+
handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS),
|
188
|
+
ack_ids - error_ack_ids,
|
189
|
+
modack: ack_deadline_seconds.nil?
|
190
|
+
ack_ids = error_ack_ids
|
191
|
+
end
|
192
|
+
perform_retry_async ack_ids, ack_deadline_seconds
|
193
|
+
end
|
194
|
+
|
195
|
+
def parse_error error, modack: false
|
196
|
+
metadata = error.error_metadata
|
197
|
+
return if metadata.nil?
|
198
|
+
permanent_failures, temporary_failures = metadata.partition do |_, v|
|
199
|
+
v.include? PERMANENT_FAILURE
|
200
|
+
end.map(&:to_h)
|
201
|
+
unless permanent_failures.empty?
|
202
|
+
handle_callback construct_result(error),
|
203
|
+
permanent_failures.keys.map(&:to_s),
|
204
|
+
modack: modack
|
205
|
+
end
|
206
|
+
temporary_failures.keys.map(&:to_s) unless temporary_failures.empty?
|
207
|
+
end
|
208
|
+
|
209
|
+
def construct_result error
|
210
|
+
case error
|
211
|
+
when Google::Cloud::PermissionDeniedError
|
212
|
+
AcknowledgeResult.new AcknowledgeResult::PERMISSION_DENIED, error
|
213
|
+
when Google::Cloud::FailedPreconditionError
|
214
|
+
AcknowledgeResult.new AcknowledgeResult::FAILED_PRECONDITION, error
|
215
|
+
when Google::Cloud::InvalidArgumentError
|
216
|
+
AcknowledgeResult.new AcknowledgeResult::INVALID_ACK_ID, error
|
217
|
+
else
|
218
|
+
AcknowledgeResult.new AcknowledgeResult::OTHER, error
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def handle_callback result, ack_ids, modack: false
|
223
|
+
ack_ids.each do |ack_id|
|
224
|
+
callback = modack ? @modack_callback_register[ack_id] : @ack_callback_register[ack_id]
|
225
|
+
perform_callback_async result, callback unless callback.nil?
|
226
|
+
end
|
227
|
+
synchronize do
|
228
|
+
if modack
|
229
|
+
@modack_callback_register.delete_if { |ack_id, _| ack_ids.include? ack_id }
|
230
|
+
else
|
231
|
+
@ack_callback_register.delete_if { |ack_id, _| ack_ids.include? ack_id }
|
232
|
+
end
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
def perform_callback_async result, callback
|
237
|
+
return unless retry_thread_pool.running?
|
238
|
+
Concurrent::Promises.future_on(
|
239
|
+
callback_thread_pool, result, callback, &method(:perform_callback_sync)
|
240
|
+
)
|
241
|
+
end
|
242
|
+
|
243
|
+
def perform_callback_sync result, callback
|
244
|
+
begin
|
245
|
+
callback.call result unless stopped?
|
246
|
+
rescue StandardError => e
|
247
|
+
@subscriber.error! e
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
251
|
+
def perform_retry_async ack_ids, ack_deadline_seconds = nil
|
252
|
+
return unless retry_thread_pool.running?
|
253
|
+
Concurrent::Promises.future_on(
|
254
|
+
retry_thread_pool, ack_ids, ack_deadline_seconds, &method(:retry_transient_error)
|
255
|
+
)
|
256
|
+
end
|
257
|
+
|
258
|
+
def retry_transient_error ack_ids, ack_deadline_seconds
|
259
|
+
if ack_deadline_seconds.nil?
|
260
|
+
retry_request ack_ids, ack_deadline_seconds.nil? do |retry_ack_ids|
|
261
|
+
@subscriber.service.acknowledge subscription_name, *retry_ack_ids
|
262
|
+
handle_callback AcknowledgeResult.new AcknowledgeResult::SUCCESS, retry_ack_ids
|
263
|
+
end
|
264
|
+
else
|
265
|
+
retry_request ack_ids, ack_deadline_seconds.nil? do |retry_ack_ids|
|
266
|
+
@subscriber.service.modify_ack_deadline subscription_name, retry_ack_ids, ack_deadline_seconds
|
267
|
+
handle_callback AcknowledgeResult.new AcknowledgeResult::SUCCESS, retry_ack_ids, modack: true
|
268
|
+
end
|
269
|
+
end
|
270
|
+
end
|
271
|
+
|
272
|
+
def retry_request ack_ids, modack
|
273
|
+
begin
|
274
|
+
Retriable.retriable tries: MAX_TRIES,
|
275
|
+
base_interval: BASE_INTERVAL,
|
276
|
+
max_interval: MAX_INTERVAL,
|
277
|
+
multiplier: MULTIPLIER,
|
278
|
+
max_elapsed_time: MAX_RETRY_DURATION,
|
279
|
+
on: EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS do
|
280
|
+
return if ack_ids.nil?
|
281
|
+
begin
|
282
|
+
yield ack_ids
|
283
|
+
rescue Google::Cloud::InvalidArgumentError => e
|
284
|
+
error_ack_ids = parse_error e.error_metadata, modack: modack
|
285
|
+
unless error_ack_ids.nil?
|
286
|
+
handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS),
|
287
|
+
ack_ids - error_ack_ids,
|
288
|
+
modack: modack
|
289
|
+
end
|
290
|
+
ack_ids = error_ack_ids
|
291
|
+
raise e
|
292
|
+
end
|
293
|
+
end
|
294
|
+
rescue StandardError => e
|
295
|
+
handle_callback e, ack_ids, modack: modack
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
132
299
|
def flush_requests!
|
133
300
|
prev_reg =
|
134
301
|
synchronize do
|
@@ -76,6 +76,10 @@ module Google
|
|
76
76
|
# @private Implementation attributes.
|
77
77
|
attr_reader :stream_pool, :thread_pool, :buffer, :service
|
78
78
|
|
79
|
+
##
|
80
|
+
# @private Implementation attributes.
|
81
|
+
attr_accessor :exactly_once_delivery_enabled
|
82
|
+
|
79
83
|
##
|
80
84
|
# @private Create an empty {Subscriber} object.
|
81
85
|
def initialize subscription_name, callback, deadline: nil, message_ordering: nil, streams: nil, inventory: nil,
|
@@ -91,6 +95,7 @@ module Google
|
|
91
95
|
@message_ordering = message_ordering
|
92
96
|
@callback_threads = Integer(threads[:callback] || 8)
|
93
97
|
@push_threads = Integer(threads[:push] || 4)
|
98
|
+
@exactly_once_delivery_enabled = nil
|
94
99
|
|
95
100
|
@service = service
|
96
101
|
|
@@ -364,7 +364,7 @@ module Google
|
|
364
364
|
##
|
365
365
|
# Creates a new {Subscription} object on the current Topic.
|
366
366
|
#
|
367
|
-
# @
|
367
|
+
# @option options [String] subscription_name Name of the new subscription. Required.
|
368
368
|
# The value can be a simple subscription ID (relative name), in which
|
369
369
|
# case the current project ID will be supplied, or a fully-qualified
|
370
370
|
# subscription name in the form
|
@@ -375,26 +375,27 @@ module Google
|
|
375
375
|
# underscores (`_`), periods (`.`), tildes (`~`), plus (`+`) or percent
|
376
376
|
# signs (`%`). It must be between 3 and 255 characters in length, and
|
377
377
|
# it must not start with `goog`.
|
378
|
-
# @
|
378
|
+
# @option options [Integer] deadline The maximum number of seconds after a
|
379
379
|
# subscriber receives a message before the subscriber should
|
380
380
|
# acknowledge the message.
|
381
|
-
# @
|
381
|
+
# @option options [Boolean] retain_acked Indicates whether to retain acknowledged
|
382
382
|
# messages. If `true`, then messages are not expunged from the
|
383
383
|
# subscription's backlog, even if they are acknowledged, until they
|
384
384
|
# fall out of the `retention` window. Default is `false`.
|
385
|
-
# @
|
385
|
+
# @option options [Numeric] retention How long to retain unacknowledged messages
|
386
386
|
# in the subscription's backlog, from the moment a message is
|
387
387
|
# published. If `retain_acked` is `true`, then this also configures
|
388
388
|
# the retention of acknowledged messages, and thus configures how far
|
389
389
|
# back in time a {Subscription#seek} can be done. Cannot be more than
|
390
390
|
# 604,800 seconds (7 days) or less than 600 seconds (10 minutes).
|
391
391
|
# Default is 604,800 seconds (7 days).
|
392
|
-
# @
|
392
|
+
# @option options [String] endpoint A URL locating the endpoint to which messages
|
393
393
|
# should be pushed. The parameters `push_config` and `endpoint` should not both be provided.
|
394
|
-
# @
|
395
|
-
#
|
396
|
-
#
|
397
|
-
#
|
394
|
+
# @option options [Google::Cloud::PubSub::Subscription::PushConfig] push_config
|
395
|
+
# The configuration for a push delivery endpoint that should contain the endpoint,
|
396
|
+
# and can contain authentication data (OIDC token authentication).
|
397
|
+
# The parameters `push_config` and `endpoint` should not both be provided.
|
398
|
+
# @option options [Hash] labels A hash of user-provided labels associated with
|
398
399
|
# the subscription. You can use these to organize and group your
|
399
400
|
# subscriptions. Label keys and values can be no longer than 63
|
400
401
|
# characters, can only contain lowercase letters, numeric characters,
|
@@ -402,25 +403,29 @@ module Google
|
|
402
403
|
# values are optional. Label keys must start with a letter and each
|
403
404
|
# label in the list must have a different key. See [Creating and
|
404
405
|
# Managing Labels](https://cloud.google.com/pubsub/docs/labels).
|
405
|
-
# @
|
406
|
+
# @option options [Boolean] message_ordering Whether to enable message ordering
|
406
407
|
# on the subscription.
|
407
|
-
# @
|
408
|
-
# {Message} instances whose `attributes` field
|
408
|
+
# @option options [String] filter An expression written in the Cloud Pub/Sub filter language.
|
409
|
+
# If non-empty, then only {Message} instances whose `attributes` field
|
410
|
+
# matches the filter are delivered on this subscription. If
|
409
411
|
# empty, then no messages are filtered out. Optional.
|
410
|
-
# @
|
411
|
-
#
|
412
|
+
# @option options [Topic] dead_letter_topic
|
413
|
+
# The {Topic} to which dead letter messages for the subscription should be published.
|
414
|
+
# Dead lettering is done on a best effort basis. The same message might be dead lettered multiple
|
412
415
|
# times. The Cloud Pub/Sub service account associated with the enclosing subscription's parent project (i.e.,
|
413
416
|
# `service-{project_number}@gcp-sa-pubsub.iam.gserviceaccount.com`) must have permission to Publish() to
|
414
417
|
# this topic.
|
415
418
|
#
|
416
419
|
# The operation will fail if the topic does not exist. Users should ensure that there is a subscription
|
417
420
|
# attached to this topic since messages published to a topic with no subscriptions are lost.
|
418
|
-
# @
|
419
|
-
#
|
421
|
+
# @option options [Integer] dead_letter_max_delivery_attempts
|
422
|
+
# The maximum number of delivery attempts for any message in the subscription's dead letter policy.
|
423
|
+
# Dead lettering is done on a best effort basis. The same message might
|
420
424
|
# be dead lettered multiple times. The value must be between 5 and 100. If this parameter is 0, a default
|
421
425
|
# value of 5 is used. The `dead_letter_topic` must also be set.
|
422
|
-
# @
|
423
|
-
#
|
426
|
+
# @option options [RetryPolicy] retry_policy
|
427
|
+
# A policy that specifies how Cloud Pub/Sub retries message delivery for this subscription.
|
428
|
+
# If not set, the default retry policy is applied. This generally implies that messages
|
424
429
|
# will be retried as soon as possible for healthy subscribers. Retry Policy will be triggered on NACKs or
|
425
430
|
# acknowledgement deadline exceeded events for a given message.
|
426
431
|
#
|
@@ -486,41 +491,23 @@ module Google
|
|
486
491
|
# retry_policy = Google::Cloud::PubSub::RetryPolicy.new minimum_backoff: 5, maximum_backoff: 300
|
487
492
|
# sub = topic.subscribe "my-topic-sub", retry_policy: retry_policy
|
488
493
|
#
|
489
|
-
def subscribe subscription_name,
|
490
|
-
deadline: nil,
|
491
|
-
retain_acked: false,
|
492
|
-
retention: nil,
|
493
|
-
endpoint: nil,
|
494
|
-
push_config: nil,
|
495
|
-
labels: nil,
|
496
|
-
message_ordering: nil,
|
497
|
-
filter: nil,
|
498
|
-
dead_letter_topic: nil,
|
499
|
-
dead_letter_max_delivery_attempts: nil,
|
500
|
-
retry_policy: nil
|
494
|
+
def subscribe subscription_name, **options
|
501
495
|
ensure_service!
|
502
|
-
if push_config && endpoint
|
496
|
+
if options[:push_config] && options[:endpoint]
|
503
497
|
raise ArgumentError, "endpoint and push_config were both provided. Please provide only one."
|
504
498
|
end
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
|
509
|
-
retain_acked: retain_acked,
|
510
|
-
retention: retention,
|
511
|
-
labels: labels,
|
512
|
-
message_ordering: message_ordering,
|
513
|
-
filter: filter,
|
514
|
-
dead_letter_max_delivery_attempts: dead_letter_max_delivery_attempts
|
515
|
-
}
|
499
|
+
if options[:endpoint]
|
500
|
+
options[:push_config] =
|
501
|
+
Google::Cloud::PubSub::Subscription::PushConfig.new endpoint: options[:endpoint]
|
502
|
+
end
|
516
503
|
|
517
|
-
options[:dead_letter_topic_name] = dead_letter_topic.name if dead_letter_topic
|
504
|
+
options[:dead_letter_topic_name] = options[:dead_letter_topic].name if options[:dead_letter_topic]
|
518
505
|
if options[:dead_letter_max_delivery_attempts] && !options[:dead_letter_topic_name]
|
519
506
|
# Service error message "3:Invalid resource name given (name=)." does not identify param.
|
520
507
|
raise ArgumentError, "dead_letter_topic is required with dead_letter_max_delivery_attempts"
|
521
508
|
end
|
522
|
-
options[:push_config] = push_config.to_grpc if push_config
|
523
|
-
options[:retry_policy] = retry_policy.to_grpc if retry_policy
|
509
|
+
options[:push_config] = options[:push_config].to_grpc if options[:push_config]
|
510
|
+
options[:retry_policy] = options[:retry_policy].to_grpc if options[:retry_policy]
|
524
511
|
grpc = service.create_subscription name, subscription_name, options
|
525
512
|
Subscription.from_grpc grpc, service
|
526
513
|
end
|
@@ -1078,7 +1065,7 @@ module Google
|
|
1078
1065
|
# @private New reference {Topic} object without making an HTTP request.
|
1079
1066
|
def self.from_name name, service, options = {}
|
1080
1067
|
name = service.topic_path name, options
|
1081
|
-
from_grpc(nil, service).tap do |t|
|
1068
|
+
from_grpc(nil, service, async: options[:async]).tap do |t|
|
1082
1069
|
t.instance_variable_set :@resource_name, name
|
1083
1070
|
end
|
1084
1071
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: google-cloud-pubsub
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.12.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mike Moore
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2022-
|
12
|
+
date: 2022-08-22 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: concurrent-ruby
|
@@ -45,14 +45,28 @@ dependencies:
|
|
45
45
|
requirements:
|
46
46
|
- - "~>"
|
47
47
|
- !ruby/object:Gem::Version
|
48
|
-
version: '0.
|
48
|
+
version: '0.8'
|
49
49
|
type: :runtime
|
50
50
|
prerelease: false
|
51
51
|
version_requirements: !ruby/object:Gem::Requirement
|
52
52
|
requirements:
|
53
53
|
- - "~>"
|
54
54
|
- !ruby/object:Gem::Version
|
55
|
-
version: '0.
|
55
|
+
version: '0.8'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: retriable
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - "~>"
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '3.1'
|
63
|
+
type: :runtime
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - "~>"
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '3.1'
|
56
70
|
- !ruby/object:Gem::Dependency
|
57
71
|
name: autotest-suffix
|
58
72
|
requirement: !ruby/object:Gem::Requirement
|
@@ -101,14 +115,14 @@ dependencies:
|
|
101
115
|
requirements:
|
102
116
|
- - "~>"
|
103
117
|
- !ruby/object:Gem::Version
|
104
|
-
version: '5.
|
118
|
+
version: '5.16'
|
105
119
|
type: :development
|
106
120
|
prerelease: false
|
107
121
|
version_requirements: !ruby/object:Gem::Requirement
|
108
122
|
requirements:
|
109
123
|
- - "~>"
|
110
124
|
- !ruby/object:Gem::Version
|
111
|
-
version: '5.
|
125
|
+
version: '5.16'
|
112
126
|
- !ruby/object:Gem::Dependency
|
113
127
|
name: minitest-autotest
|
114
128
|
requirement: !ruby/object:Gem::Requirement
|
@@ -227,6 +241,7 @@ files:
|
|
227
241
|
- TROUBLESHOOTING.md
|
228
242
|
- lib/google-cloud-pubsub.rb
|
229
243
|
- lib/google/cloud/pubsub.rb
|
244
|
+
- lib/google/cloud/pubsub/acknowledge_result.rb
|
230
245
|
- lib/google/cloud/pubsub/async_publisher.rb
|
231
246
|
- lib/google/cloud/pubsub/async_publisher/batch.rb
|
232
247
|
- lib/google/cloud/pubsub/batch_publisher.rb
|