google-cloud-pubsub 2.10.0 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/lib/google/cloud/pubsub/acknowledge_result.rb +79 -0
- data/lib/google/cloud/pubsub/received_message.rb +77 -6
- data/lib/google/cloud/pubsub/service.rb +30 -24
- data/lib/google/cloud/pubsub/subscriber/stream.rb +5 -4
- data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb +176 -9
- data/lib/google/cloud/pubsub/subscriber.rb +5 -0
- data/lib/google/cloud/pubsub/topic.rb +32 -45
- data/lib/google/cloud/pubsub/version.rb +1 -1
- metadata +19 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2c1517a2425dde72d0dc828b7a9547377b2c77d040ab456e75a22da05ec435a9
|
4
|
+
data.tar.gz: 8d5dcddd9253520df1192021cd9cdf6b582045aa39b32a4213870ed95a6a81db
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 23fe9089e74ffb8f578c9cf3f7122601f3e3440c3d891b80231fc0fd20c7f11128f7c6402f5be9b1e8737ddd6c18f7a1756c2ecdeae3771653506603cab25cef
|
7
|
+
data.tar.gz: c874cdff8afc7f4aeef73ee7c48b556314c2698934f63349644fe9c9e58b00dfdbb45b96c8fd58fad8218af1225c551ce72db98ee26744bc86eec9df3a267747
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,13 @@
|
|
1
1
|
# Release History
|
2
2
|
|
3
|
+
### 2.11.0 (2022-08-01)
|
4
|
+
|
5
|
+
#### Features
|
6
|
+
|
7
|
+
* create exactly once delivery enabled subscription ([#18824](https://github.com/googleapis/google-cloud-ruby/issues/18824))
|
8
|
+
* Let user register callback and get acknowledgement result ([#18702](https://github.com/googleapis/google-cloud-ruby/issues/18702))
|
9
|
+
* retry transient failures in ack/modack in timed unary buffer ([#18395](https://github.com/googleapis/google-cloud-ruby/issues/18395))
|
10
|
+
|
3
11
|
### 2.10.0 (2022-06-14)
|
4
12
|
|
5
13
|
#### Features
|
@@ -0,0 +1,79 @@
|
|
1
|
+
# Copyright 2022 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
module Google
|
17
|
+
module Cloud
|
18
|
+
module PubSub
|
19
|
+
##
|
20
|
+
# The result of a ack/nack/modack on messages.
|
21
|
+
#
|
22
|
+
# When the operation was successful the result will be marked
|
23
|
+
# {#succeeded?}. Otherwise, the result will be marked {#failed?} and the
|
24
|
+
# error raised will be availabe on {#error}.
|
25
|
+
#
|
26
|
+
class AcknowledgeResult
|
27
|
+
##
|
28
|
+
# The constants below represents the status of ack/modack operations.
|
29
|
+
# Indicates successful ack/modack
|
30
|
+
SUCCESS = 1
|
31
|
+
|
32
|
+
##
|
33
|
+
# Indicates occurence of permenant permission denied error
|
34
|
+
PERMISSION_DENIED = 2
|
35
|
+
|
36
|
+
##
|
37
|
+
# Indicates occurence of permenant failed precondition error
|
38
|
+
FAILED_PRECONDITION = 3
|
39
|
+
|
40
|
+
##
|
41
|
+
# Indicates occurence of permenant permission denied error
|
42
|
+
INVALID_ACK_ID = 4
|
43
|
+
|
44
|
+
##
|
45
|
+
# Indicates occurence of permenant uncatogorised error
|
46
|
+
OTHER = 5
|
47
|
+
|
48
|
+
##
|
49
|
+
# @return [Google::Cloud::Error] Error object of ack/modack operation
|
50
|
+
attr_reader :error
|
51
|
+
|
52
|
+
##
|
53
|
+
# @return [Numeric] Status of the ack/modack operation.
|
54
|
+
attr_reader :status
|
55
|
+
|
56
|
+
##
|
57
|
+
# @private Create an PublishResult object.
|
58
|
+
def initialize status, error = nil
|
59
|
+
@error = error
|
60
|
+
@status = status
|
61
|
+
end
|
62
|
+
|
63
|
+
##
|
64
|
+
# @return [Boolean] Whether the operation was successful.
|
65
|
+
def succeeded?
|
66
|
+
@status == SUCCESS
|
67
|
+
end
|
68
|
+
|
69
|
+
##
|
70
|
+
# @return [Boolean] Whether the operation failed.
|
71
|
+
def failed?
|
72
|
+
!succeeded?
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
Pubsub = PubSub unless const_defined? :Pubsub
|
78
|
+
end
|
79
|
+
end
|
@@ -161,6 +161,29 @@ module Google
|
|
161
161
|
##
|
162
162
|
# Acknowledges receipt of the message.
|
163
163
|
#
|
164
|
+
# @yield [callback] The block to be called when reject operation is done.
|
165
|
+
# @yieldparam [Google::Cloud::PubSub::AcknowledgeResult] Result object that contains the status and error.
|
166
|
+
#
|
167
|
+
# @example
|
168
|
+
# require "google/cloud/pubsub"
|
169
|
+
#
|
170
|
+
# pubsub = Google::Cloud::PubSub.new
|
171
|
+
#
|
172
|
+
# sub = pubsub.subscription "my-topic-sub"
|
173
|
+
# subscriber = sub.listen do |received_message|
|
174
|
+
# puts received_message.message.data
|
175
|
+
#
|
176
|
+
# received_message.acknowledge! do |result|
|
177
|
+
# puts result.status
|
178
|
+
# end
|
179
|
+
# end
|
180
|
+
#
|
181
|
+
# # Start background threads that will call block passed to listen.
|
182
|
+
# subscriber.start
|
183
|
+
#
|
184
|
+
# # Shut down the subscriber when ready to stop receiving messages.
|
185
|
+
# subscriber.stop!
|
186
|
+
#
|
164
187
|
# @example
|
165
188
|
# require "google/cloud/pubsub"
|
166
189
|
#
|
@@ -179,9 +202,9 @@ module Google
|
|
179
202
|
# # Shut down the subscriber when ready to stop receiving messages.
|
180
203
|
# subscriber.stop!
|
181
204
|
#
|
182
|
-
def acknowledge!
|
205
|
+
def acknowledge! &block
|
183
206
|
ensure_subscription!
|
184
|
-
subscription.acknowledge ack_id
|
207
|
+
subscription.acknowledge ack_id, &block
|
185
208
|
end
|
186
209
|
alias ack! acknowledge!
|
187
210
|
|
@@ -197,6 +220,30 @@ module Google
|
|
197
220
|
# seconds after the call is made. Specifying `0` may immediately make
|
198
221
|
# the message available for another pull request.
|
199
222
|
#
|
223
|
+
# @yield [callback] The block to be called when reject operation is done.
|
224
|
+
# @yieldparam [Google::Cloud::PubSub::AcknowledgeResult] Result object that contains the status and error.
|
225
|
+
#
|
226
|
+
# @example
|
227
|
+
# require "google/cloud/pubsub"
|
228
|
+
#
|
229
|
+
# pubsub = Google::Cloud::PubSub.new
|
230
|
+
#
|
231
|
+
# sub = pubsub.subscription "my-topic-sub"
|
232
|
+
# subscriber = sub.listen do |received_message|
|
233
|
+
# puts received_message.message.data
|
234
|
+
#
|
235
|
+
# # Delay for 2 minutes
|
236
|
+
# received_message.modify_ack_deadline! 120 do |result|
|
237
|
+
# puts result.status
|
238
|
+
# end
|
239
|
+
# end
|
240
|
+
#
|
241
|
+
# # Start background threads that will call block passed to listen.
|
242
|
+
# subscriber.start
|
243
|
+
#
|
244
|
+
# # Shut down the subscriber when ready to stop receiving messages.
|
245
|
+
# subscriber.stop!
|
246
|
+
#
|
200
247
|
# @example
|
201
248
|
# require "google/cloud/pubsub"
|
202
249
|
#
|
@@ -216,9 +263,9 @@ module Google
|
|
216
263
|
# # Shut down the subscriber when ready to stop receiving messages.
|
217
264
|
# subscriber.stop!
|
218
265
|
#
|
219
|
-
def modify_ack_deadline! new_deadline
|
266
|
+
def modify_ack_deadline! new_deadline, &block
|
220
267
|
ensure_subscription!
|
221
|
-
subscription.modify_ack_deadline new_deadline, ack_id
|
268
|
+
subscription.modify_ack_deadline new_deadline, ack_id, &block
|
222
269
|
end
|
223
270
|
|
224
271
|
##
|
@@ -227,6 +274,30 @@ module Google
|
|
227
274
|
#
|
228
275
|
# This will make the message available for redelivery.
|
229
276
|
#
|
277
|
+
# @yield [callback] The block to be called when reject operation is done.
|
278
|
+
# @yieldparam [Google::Cloud::PubSub::AcknowledgeResult] Result object that contains the status and error.
|
279
|
+
#
|
280
|
+
# @example
|
281
|
+
# require "google/cloud/pubsub"
|
282
|
+
#
|
283
|
+
# pubsub = Google::Cloud::PubSub.new
|
284
|
+
#
|
285
|
+
# sub = pubsub.subscription "my-topic-sub"
|
286
|
+
# subscriber = sub.listen do |received_message|
|
287
|
+
# puts received_message.message.data
|
288
|
+
#
|
289
|
+
# # Release message back to the API.
|
290
|
+
# received_message.reject! do |result|
|
291
|
+
# puts result.status
|
292
|
+
# end
|
293
|
+
# end
|
294
|
+
#
|
295
|
+
# # Start background threads that will call block passed to listen.
|
296
|
+
# subscriber.start
|
297
|
+
#
|
298
|
+
# # Shut down the subscriber when ready to stop receiving messages.
|
299
|
+
# subscriber.stop!
|
300
|
+
#
|
230
301
|
# @example
|
231
302
|
# require "google/cloud/pubsub"
|
232
303
|
#
|
@@ -246,8 +317,8 @@ module Google
|
|
246
317
|
# # Shut down the subscriber when ready to stop receiving messages.
|
247
318
|
# subscriber.stop!
|
248
319
|
#
|
249
|
-
def reject!
|
250
|
-
modify_ack_deadline! 0
|
320
|
+
def reject! &block
|
321
|
+
modify_ack_deadline! 0, &block
|
251
322
|
end
|
252
323
|
alias nack! reject!
|
253
324
|
alias ignore! reject!
|
@@ -204,18 +204,8 @@ module Google
|
|
204
204
|
##
|
205
205
|
# Creates a subscription on a given topic for a given subscriber.
|
206
206
|
def create_subscription topic, subscription_name, options = {}
|
207
|
-
|
208
|
-
|
209
|
-
topic: topic_path(topic),
|
210
|
-
push_config: options[:push_config],
|
211
|
-
ack_deadline_seconds: options[:deadline],
|
212
|
-
retain_acked_messages: options[:retain_acked],
|
213
|
-
message_retention_duration: Convert.number_to_duration(options[:retention]),
|
214
|
-
labels: options[:labels],
|
215
|
-
enable_message_ordering: options[:message_ordering],
|
216
|
-
filter: options[:filter],
|
217
|
-
dead_letter_policy: dead_letter_policy(options),
|
218
|
-
retry_policy: options[:retry_policy]
|
207
|
+
updated_option = construct_create_subscription_options topic, subscription_name, options
|
208
|
+
subscriber.create_subscription(**updated_option)
|
219
209
|
end
|
220
210
|
|
221
211
|
def update_subscription subscription_obj, *fields
|
@@ -255,11 +245,7 @@ module Google
|
|
255
245
|
##
|
256
246
|
# Acknowledges receipt of a message.
|
257
247
|
def acknowledge subscription, *ack_ids
|
258
|
-
|
259
|
-
subscriber.acknowledge subscription: subscription_path(subscription), ack_ids: ack_ids
|
260
|
-
rescue StandardError => e
|
261
|
-
raise e unless e.cause.is_a? GRPC::BadStatus
|
262
|
-
end
|
248
|
+
subscriber.acknowledge subscription: subscription_path(subscription), ack_ids: ack_ids
|
263
249
|
end
|
264
250
|
|
265
251
|
##
|
@@ -279,13 +265,9 @@ module Google
|
|
279
265
|
##
|
280
266
|
# Modifies the ack deadline for a specific message.
|
281
267
|
def modify_ack_deadline subscription, ids, deadline
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
ack_deadline_seconds: deadline
|
286
|
-
rescue StandardError => e
|
287
|
-
raise e unless e.cause.is_a? GRPC::BadStatus
|
288
|
-
end
|
268
|
+
subscriber.modify_ack_deadline subscription: subscription_path(subscription),
|
269
|
+
ack_ids: Array(ids),
|
270
|
+
ack_deadline_seconds: deadline
|
289
271
|
end
|
290
272
|
|
291
273
|
##
|
@@ -496,6 +478,30 @@ module Google
|
|
496
478
|
end
|
497
479
|
policy
|
498
480
|
end
|
481
|
+
|
482
|
+
private
|
483
|
+
|
484
|
+
def construct_create_subscription_options topic, subscription_name, options
|
485
|
+
excess_options = [:deadline,
|
486
|
+
:retention,
|
487
|
+
:retain_acked,
|
488
|
+
:message_ordering,
|
489
|
+
:endpoint,
|
490
|
+
:dead_letter_topic_name,
|
491
|
+
:dead_letter_max_delivery_attempts,
|
492
|
+
:dead_letter_topic]
|
493
|
+
|
494
|
+
new_options = options.filter { |k, v| !v.nil? && !excess_options.include?(k) }
|
495
|
+
new_options[:name] = subscription_path subscription_name, options
|
496
|
+
new_options[:topic] = topic_path topic
|
497
|
+
new_options[:message_retention_duration] = Convert.number_to_duration options[:retention]
|
498
|
+
new_options[:dead_letter_policy] = dead_letter_policy options
|
499
|
+
new_options[:ack_deadline_seconds] = options[:deadline]
|
500
|
+
new_options[:retain_acked_messages] = options[:retain_acked]
|
501
|
+
new_options[:enable_message_ordering] = options[:message_ordering]
|
502
|
+
|
503
|
+
new_options.compact
|
504
|
+
end
|
499
505
|
end
|
500
506
|
end
|
501
507
|
|
@@ -134,13 +134,13 @@ module Google
|
|
134
134
|
|
135
135
|
##
|
136
136
|
# @private
|
137
|
-
def acknowledge *messages
|
137
|
+
def acknowledge *messages, &callback
|
138
138
|
ack_ids = coerce_ack_ids messages
|
139
139
|
return true if ack_ids.empty?
|
140
140
|
|
141
141
|
synchronize do
|
142
142
|
@inventory.remove ack_ids
|
143
|
-
@subscriber.buffer.acknowledge ack_ids
|
143
|
+
@subscriber.buffer.acknowledge ack_ids, callback
|
144
144
|
end
|
145
145
|
|
146
146
|
true
|
@@ -148,13 +148,13 @@ module Google
|
|
148
148
|
|
149
149
|
##
|
150
150
|
# @private
|
151
|
-
def modify_ack_deadline deadline, *messages
|
151
|
+
def modify_ack_deadline deadline, *messages, &callback
|
152
152
|
mod_ack_ids = coerce_ack_ids messages
|
153
153
|
return true if mod_ack_ids.empty?
|
154
154
|
|
155
155
|
synchronize do
|
156
156
|
@inventory.remove mod_ack_ids
|
157
|
-
@subscriber.buffer.modify_ack_deadline deadline, mod_ack_ids
|
157
|
+
@subscriber.buffer.modify_ack_deadline deadline, mod_ack_ids, callback
|
158
158
|
end
|
159
159
|
|
160
160
|
true
|
@@ -253,6 +253,7 @@ module Google
|
|
253
253
|
synchronize do
|
254
254
|
update_min_duration_per_lease_extension new_exactly_once_delivery_enabled
|
255
255
|
@exactly_once_delivery_enabled = new_exactly_once_delivery_enabled unless new_exactly_once_delivery_enabled.nil?
|
256
|
+
@subscriber.exactly_once_delivery_enabled = @exactly_once_delivery_enabled
|
256
257
|
|
257
258
|
# Create receipt of received messages reception
|
258
259
|
@subscriber.buffer.modify_ack_deadline @subscriber.deadline, response.received_messages.map(&:ack_id)
|
@@ -14,7 +14,10 @@
|
|
14
14
|
|
15
15
|
|
16
16
|
require "concurrent"
|
17
|
+
require "google/cloud/errors"
|
18
|
+
require "google/cloud/pubsub/acknowledge_result"
|
17
19
|
require "monitor"
|
20
|
+
require "retriable"
|
18
21
|
|
19
22
|
module Google
|
20
23
|
module Cloud
|
@@ -27,6 +30,21 @@ module Google
|
|
27
30
|
|
28
31
|
attr_reader :max_bytes
|
29
32
|
attr_reader :interval
|
33
|
+
attr_reader :retry_thread_pool
|
34
|
+
attr_reader :callback_thread_pool
|
35
|
+
|
36
|
+
PERMANENT_FAILURE = "PERMANENT_FAILURE".freeze
|
37
|
+
# Google::Cloud::Unavailable error is already retried at gapic level
|
38
|
+
RETRIABLE_ERRORS = [Google::Cloud::CanceledError,
|
39
|
+
Google::Cloud::DeadlineExceededError,
|
40
|
+
Google::Cloud::InternalError,
|
41
|
+
Google::Cloud::ResourceExhaustedError,
|
42
|
+
Google::Cloud::InvalidArgumentError].freeze
|
43
|
+
MAX_RETRY_DURATION = 600 # 600s since the server allows ack/modacks for 10 mins max
|
44
|
+
MAX_TRIES = 15
|
45
|
+
BASE_INTERVAL = 1
|
46
|
+
MAX_INTERVAL = 64
|
47
|
+
MULTIPLIER = 2
|
30
48
|
|
31
49
|
def initialize subscriber, max_bytes: 500_000, interval: 1.0
|
32
50
|
super() # to init MonitorMixin
|
@@ -40,30 +58,37 @@ module Google
|
|
40
58
|
# entry.
|
41
59
|
@register = {}
|
42
60
|
|
61
|
+
@ack_callback_register = {}
|
62
|
+
@modack_callback_register = {}
|
63
|
+
|
64
|
+
@retry_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
|
65
|
+
@callback_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
|
43
66
|
@task = Concurrent::TimerTask.new execution_interval: interval do
|
44
67
|
flush!
|
45
68
|
end
|
46
69
|
end
|
47
70
|
|
48
|
-
def acknowledge ack_ids
|
71
|
+
def acknowledge ack_ids, callback = nil
|
49
72
|
return if ack_ids.empty?
|
50
73
|
|
51
74
|
synchronize do
|
52
75
|
ack_ids.each do |ack_id|
|
53
76
|
# ack has no deadline set, use :ack indicate it is an ack
|
54
77
|
@register[ack_id] = :ack
|
78
|
+
@ack_callback_register[ack_id] = callback unless callback.nil?
|
55
79
|
end
|
56
80
|
end
|
57
81
|
|
58
82
|
true
|
59
83
|
end
|
60
84
|
|
61
|
-
def modify_ack_deadline deadline, ack_ids
|
85
|
+
def modify_ack_deadline deadline, ack_ids, callback = nil
|
62
86
|
return if ack_ids.empty?
|
63
87
|
|
64
88
|
synchronize do
|
65
89
|
ack_ids.each do |ack_id|
|
66
90
|
@register[ack_id] = deadline
|
91
|
+
@modack_callback_register[ack_id] = callback unless callback.nil?
|
67
92
|
end
|
68
93
|
end
|
69
94
|
|
@@ -90,20 +115,46 @@ module Google
|
|
90
115
|
|
91
116
|
# Perform the RCP calls concurrently
|
92
117
|
with_threadpool do |pool|
|
93
|
-
requests
|
94
|
-
|
118
|
+
make_acknowledge_request requests, pool
|
119
|
+
make_modack_request requests, pool
|
120
|
+
end
|
121
|
+
|
122
|
+
true
|
123
|
+
end
|
124
|
+
|
125
|
+
def make_acknowledge_request requests, pool
|
126
|
+
requests[:acknowledge].each do |ack_req|
|
127
|
+
add_future pool do
|
128
|
+
begin
|
95
129
|
@subscriber.service.acknowledge ack_req.subscription, *ack_req.ack_ids
|
130
|
+
handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS), ack_req.ack_ids
|
131
|
+
rescue *RETRIABLE_ERRORS => e
|
132
|
+
handle_failure e, ack_req.ack_ids if @subscriber.exactly_once_delivery_enabled
|
133
|
+
rescue StandardError => e
|
134
|
+
handle_callback construct_result(e), ack_req.ack_ids
|
96
135
|
end
|
97
136
|
end
|
98
|
-
|
99
|
-
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
def make_modack_request requests, pool
|
141
|
+
requests[:modify_ack_deadline].each do |mod_ack_req|
|
142
|
+
add_future pool do
|
143
|
+
begin
|
100
144
|
@subscriber.service.modify_ack_deadline mod_ack_req.subscription, mod_ack_req.ack_ids,
|
101
145
|
mod_ack_req.ack_deadline_seconds
|
146
|
+
handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS),
|
147
|
+
mod_ack_req.ack_ids,
|
148
|
+
modack: true
|
149
|
+
rescue *RETRIABLE_ERRORS => e
|
150
|
+
if @subscriber.exactly_once_delivery_enabled
|
151
|
+
handle_failure e, mod_ack_req.ack_ids, mod_ack_req.ack_deadline_seconds
|
152
|
+
end
|
153
|
+
rescue StandardError => e
|
154
|
+
handle_callback construct_result(e), mod_ack_req.ack_ids, modack: true
|
102
155
|
end
|
103
156
|
end
|
104
157
|
end
|
105
|
-
|
106
|
-
true
|
107
158
|
end
|
108
159
|
|
109
160
|
def start
|
@@ -114,8 +165,9 @@ module Google
|
|
114
165
|
|
115
166
|
def stop
|
116
167
|
@task.shutdown
|
168
|
+
@retry_thread_pool.shutdown
|
169
|
+
@callback_thread_pool.shutdown
|
117
170
|
flush!
|
118
|
-
|
119
171
|
self
|
120
172
|
end
|
121
173
|
|
@@ -129,6 +181,121 @@ module Google
|
|
129
181
|
|
130
182
|
private
|
131
183
|
|
184
|
+
def handle_failure error, ack_ids, ack_deadline_seconds = nil
|
185
|
+
error_ack_ids = parse_error error, modack: ack_deadline_seconds.nil?
|
186
|
+
unless error_ack_ids.nil?
|
187
|
+
handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS),
|
188
|
+
ack_ids - error_ack_ids,
|
189
|
+
modack: ack_deadline_seconds.nil?
|
190
|
+
ack_ids = error_ack_ids
|
191
|
+
end
|
192
|
+
perform_retry_async ack_ids, ack_deadline_seconds
|
193
|
+
end
|
194
|
+
|
195
|
+
def parse_error error, modack: false
|
196
|
+
metadata = error.error_metadata
|
197
|
+
return if metadata.nil?
|
198
|
+
permanent_failures, temporary_failures = metadata.partition do |_, v|
|
199
|
+
v.include? PERMANENT_FAILURE
|
200
|
+
end.map(&:to_h)
|
201
|
+
unless permanent_failures.empty?
|
202
|
+
handle_callback construct_result(error),
|
203
|
+
permanent_failures.keys.map(&:to_s),
|
204
|
+
modack: modack
|
205
|
+
end
|
206
|
+
temporary_failures.keys.map(&:to_s) unless temporary_failures.empty?
|
207
|
+
end
|
208
|
+
|
209
|
+
def construct_result error
|
210
|
+
case error
|
211
|
+
when Google::Cloud::PermissionDeniedError
|
212
|
+
AcknowledgeResult.new AcknowledgeResult::PERMISSION_DENIED, error
|
213
|
+
when Google::Cloud::FailedPreconditionError
|
214
|
+
AcknowledgeResult.new AcknowledgeResult::FAILED_PRECONDITION, error
|
215
|
+
when Google::Cloud::InvalidArgumentError
|
216
|
+
AcknowledgeResult.new AcknowledgeResult::INVALID_ACK_ID, error
|
217
|
+
else
|
218
|
+
AcknowledgeResult.new AcknowledgeResult::OTHER, error
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
def handle_callback result, ack_ids, modack: false
|
223
|
+
ack_ids.each do |ack_id|
|
224
|
+
callback = modack ? @modack_callback_register[ack_id] : @ack_callback_register[ack_id]
|
225
|
+
perform_callback_async result, callback unless callback.nil?
|
226
|
+
end
|
227
|
+
synchronize do
|
228
|
+
if modack
|
229
|
+
@modack_callback_register.delete_if { |ack_id, _| ack_ids.include? ack_id }
|
230
|
+
else
|
231
|
+
@ack_callback_register.delete_if { |ack_id, _| ack_ids.include? ack_id }
|
232
|
+
end
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
def perform_callback_async result, callback
|
237
|
+
return unless retry_thread_pool.running?
|
238
|
+
Concurrent::Promises.future_on(
|
239
|
+
callback_thread_pool, result, callback, &method(:perform_callback_sync)
|
240
|
+
)
|
241
|
+
end
|
242
|
+
|
243
|
+
def perform_callback_sync result, callback
|
244
|
+
begin
|
245
|
+
callback.call result unless stopped?
|
246
|
+
rescue StandardError => e
|
247
|
+
@subscriber.error! e
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
251
|
+
def perform_retry_async ack_ids, ack_deadline_seconds = nil
|
252
|
+
return unless retry_thread_pool.running?
|
253
|
+
Concurrent::Promises.future_on(
|
254
|
+
retry_thread_pool, ack_ids, ack_deadline_seconds, &method(:retry_transient_error)
|
255
|
+
)
|
256
|
+
end
|
257
|
+
|
258
|
+
def retry_transient_error ack_ids, ack_deadline_seconds
|
259
|
+
if ack_deadline_seconds.nil?
|
260
|
+
retry_request ack_ids, ack_deadline_seconds.nil? do |retry_ack_ids|
|
261
|
+
@subscriber.service.acknowledge subscription_name, *retry_ack_ids
|
262
|
+
handle_callback AcknowledgeResult.new AcknowledgeResult::SUCCESS, retry_ack_ids
|
263
|
+
end
|
264
|
+
else
|
265
|
+
retry_request ack_ids, ack_deadline_seconds.nil? do |retry_ack_ids|
|
266
|
+
@subscriber.service.modify_ack_deadline subscription_name, retry_ack_ids, ack_deadline_seconds
|
267
|
+
handle_callback AcknowledgeResult.new AcknowledgeResult::SUCCESS, retry_ack_ids, modack: true
|
268
|
+
end
|
269
|
+
end
|
270
|
+
end
|
271
|
+
|
272
|
+
def retry_request ack_ids, modack
|
273
|
+
begin
|
274
|
+
Retriable.retriable tries: MAX_TRIES,
|
275
|
+
base_interval: BASE_INTERVAL,
|
276
|
+
max_interval: MAX_INTERVAL,
|
277
|
+
multiplier: MULTIPLIER,
|
278
|
+
max_elapsed_time: MAX_RETRY_DURATION,
|
279
|
+
on: RETRIABLE_ERRORS do
|
280
|
+
return if ack_ids.nil?
|
281
|
+
begin
|
282
|
+
yield ack_ids
|
283
|
+
rescue Google::Cloud::InvalidArgumentError => e
|
284
|
+
error_ack_ids = parse_error e.error_metadata, modack: modack
|
285
|
+
unless error_ack_ids.nil?
|
286
|
+
handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS),
|
287
|
+
ack_ids - error_ack_ids,
|
288
|
+
modack: modack
|
289
|
+
end
|
290
|
+
ack_ids = error_ack_ids
|
291
|
+
raise e
|
292
|
+
end
|
293
|
+
end
|
294
|
+
rescue StandardError => e
|
295
|
+
handle_callback e, ack_ids, modack: modack
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
132
299
|
def flush_requests!
|
133
300
|
prev_reg =
|
134
301
|
synchronize do
|
@@ -76,6 +76,10 @@ module Google
|
|
76
76
|
# @private Implementation attributes.
|
77
77
|
attr_reader :stream_pool, :thread_pool, :buffer, :service
|
78
78
|
|
79
|
+
##
|
80
|
+
# @private Implementation attributes.
|
81
|
+
attr_accessor :exactly_once_delivery_enabled
|
82
|
+
|
79
83
|
##
|
80
84
|
# @private Create an empty {Subscriber} object.
|
81
85
|
def initialize subscription_name, callback, deadline: nil, message_ordering: nil, streams: nil, inventory: nil,
|
@@ -91,6 +95,7 @@ module Google
|
|
91
95
|
@message_ordering = message_ordering
|
92
96
|
@callback_threads = Integer(threads[:callback] || 8)
|
93
97
|
@push_threads = Integer(threads[:push] || 4)
|
98
|
+
@exactly_once_delivery_enabled = nil
|
94
99
|
|
95
100
|
@service = service
|
96
101
|
|
@@ -364,7 +364,7 @@ module Google
|
|
364
364
|
##
|
365
365
|
# Creates a new {Subscription} object on the current Topic.
|
366
366
|
#
|
367
|
-
# @
|
367
|
+
# @option options [String] subscription_name Name of the new subscription. Required.
|
368
368
|
# The value can be a simple subscription ID (relative name), in which
|
369
369
|
# case the current project ID will be supplied, or a fully-qualified
|
370
370
|
# subscription name in the form
|
@@ -375,26 +375,27 @@ module Google
|
|
375
375
|
# underscores (`_`), periods (`.`), tildes (`~`), plus (`+`) or percent
|
376
376
|
# signs (`%`). It must be between 3 and 255 characters in length, and
|
377
377
|
# it must not start with `goog`.
|
378
|
-
# @
|
378
|
+
# @option options [Integer] deadline The maximum number of seconds after a
|
379
379
|
# subscriber receives a message before the subscriber should
|
380
380
|
# acknowledge the message.
|
381
|
-
# @
|
381
|
+
# @option options [Boolean] retain_acked Indicates whether to retain acknowledged
|
382
382
|
# messages. If `true`, then messages are not expunged from the
|
383
383
|
# subscription's backlog, even if they are acknowledged, until they
|
384
384
|
# fall out of the `retention` window. Default is `false`.
|
385
|
-
# @
|
385
|
+
# @option options [Numeric] retention How long to retain unacknowledged messages
|
386
386
|
# in the subscription's backlog, from the moment a message is
|
387
387
|
# published. If `retain_acked` is `true`, then this also configures
|
388
388
|
# the retention of acknowledged messages, and thus configures how far
|
389
389
|
# back in time a {Subscription#seek} can be done. Cannot be more than
|
390
390
|
# 604,800 seconds (7 days) or less than 600 seconds (10 minutes).
|
391
391
|
# Default is 604,800 seconds (7 days).
|
392
|
-
# @
|
392
|
+
# @option options [String] endpoint A URL locating the endpoint to which messages
|
393
393
|
# should be pushed. The parameters `push_config` and `endpoint` should not both be provided.
|
394
|
-
# @
|
395
|
-
#
|
396
|
-
#
|
397
|
-
#
|
394
|
+
# @option options [Google::Cloud::PubSub::Subscription::PushConfig] push_config
|
395
|
+
# The configuration for a push delivery endpoint that should contain the endpoint,
|
396
|
+
# and can contain authentication data (OIDC token authentication).
|
397
|
+
# The parameters `push_config` and `endpoint` should not both be provided.
|
398
|
+
# @option options [Hash] labels A hash of user-provided labels associated with
|
398
399
|
# the subscription. You can use these to organize and group your
|
399
400
|
# subscriptions. Label keys and values can be no longer than 63
|
400
401
|
# characters, can only contain lowercase letters, numeric characters,
|
@@ -402,25 +403,29 @@ module Google
|
|
402
403
|
# values are optional. Label keys must start with a letter and each
|
403
404
|
# label in the list must have a different key. See [Creating and
|
404
405
|
# Managing Labels](https://cloud.google.com/pubsub/docs/labels).
|
405
|
-
# @
|
406
|
+
# @option options [Boolean] message_ordering Whether to enable message ordering
|
406
407
|
# on the subscription.
|
407
|
-
# @
|
408
|
-
# {Message} instances whose `attributes` field
|
408
|
+
# @option options [String] filter An expression written in the Cloud Pub/Sub filter language.
|
409
|
+
# If non-empty, then only {Message} instances whose `attributes` field
|
410
|
+
# matches the filter are delivered on this subscription. If
|
409
411
|
# empty, then no messages are filtered out. Optional.
|
410
|
-
# @
|
411
|
-
#
|
412
|
+
# @option options [Topic] dead_letter_topic
|
413
|
+
# The {Topic} to which dead letter messages for the subscription should be published.
|
414
|
+
# Dead lettering is done on a best effort basis. The same message might be dead lettered multiple
|
412
415
|
# times. The Cloud Pub/Sub service account associated with the enclosing subscription's parent project (i.e.,
|
413
416
|
# `service-{project_number}@gcp-sa-pubsub.iam.gserviceaccount.com`) must have permission to Publish() to
|
414
417
|
# this topic.
|
415
418
|
#
|
416
419
|
# The operation will fail if the topic does not exist. Users should ensure that there is a subscription
|
417
420
|
# attached to this topic since messages published to a topic with no subscriptions are lost.
|
418
|
-
# @
|
419
|
-
#
|
421
|
+
# @option options [Integer] dead_letter_max_delivery_attempts
|
422
|
+
# The maximum number of delivery attempts for any message in the subscription's dead letter policy.
|
423
|
+
# Dead lettering is done on a best effort basis. The same message might
|
420
424
|
# be dead lettered multiple times. The value must be between 5 and 100. If this parameter is 0, a default
|
421
425
|
# value of 5 is used. The `dead_letter_topic` must also be set.
|
422
|
-
# @
|
423
|
-
#
|
426
|
+
# @option options [RetryPolicy] retry_policy
|
427
|
+
# A policy that specifies how Cloud Pub/Sub retries message delivery for this subscription.
|
428
|
+
# If not set, the default retry policy is applied. This generally implies that messages
|
424
429
|
# will be retried as soon as possible for healthy subscribers. Retry Policy will be triggered on NACKs or
|
425
430
|
# acknowledgement deadline exceeded events for a given message.
|
426
431
|
#
|
@@ -486,41 +491,23 @@ module Google
|
|
486
491
|
# retry_policy = Google::Cloud::PubSub::RetryPolicy.new minimum_backoff: 5, maximum_backoff: 300
|
487
492
|
# sub = topic.subscribe "my-topic-sub", retry_policy: retry_policy
|
488
493
|
#
|
489
|
-
def subscribe subscription_name,
|
490
|
-
deadline: nil,
|
491
|
-
retain_acked: false,
|
492
|
-
retention: nil,
|
493
|
-
endpoint: nil,
|
494
|
-
push_config: nil,
|
495
|
-
labels: nil,
|
496
|
-
message_ordering: nil,
|
497
|
-
filter: nil,
|
498
|
-
dead_letter_topic: nil,
|
499
|
-
dead_letter_max_delivery_attempts: nil,
|
500
|
-
retry_policy: nil
|
494
|
+
def subscribe subscription_name, **options
|
501
495
|
ensure_service!
|
502
|
-
if push_config && endpoint
|
496
|
+
if options[:push_config] && options[:endpoint]
|
503
497
|
raise ArgumentError, "endpoint and push_config were both provided. Please provide only one."
|
504
498
|
end
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
|
509
|
-
retain_acked: retain_acked,
|
510
|
-
retention: retention,
|
511
|
-
labels: labels,
|
512
|
-
message_ordering: message_ordering,
|
513
|
-
filter: filter,
|
514
|
-
dead_letter_max_delivery_attempts: dead_letter_max_delivery_attempts
|
515
|
-
}
|
499
|
+
if options[:endpoint]
|
500
|
+
options[:push_config] =
|
501
|
+
Google::Cloud::PubSub::Subscription::PushConfig.new endpoint: options[:endpoint]
|
502
|
+
end
|
516
503
|
|
517
|
-
options[:dead_letter_topic_name] = dead_letter_topic.name if dead_letter_topic
|
504
|
+
options[:dead_letter_topic_name] = options[:dead_letter_topic].name if options[:dead_letter_topic]
|
518
505
|
if options[:dead_letter_max_delivery_attempts] && !options[:dead_letter_topic_name]
|
519
506
|
# Service error message "3:Invalid resource name given (name=)." does not identify param.
|
520
507
|
raise ArgumentError, "dead_letter_topic is required with dead_letter_max_delivery_attempts"
|
521
508
|
end
|
522
|
-
options[:push_config] = push_config.to_grpc if push_config
|
523
|
-
options[:retry_policy] = retry_policy.to_grpc if retry_policy
|
509
|
+
options[:push_config] = options[:push_config].to_grpc if options[:push_config]
|
510
|
+
options[:retry_policy] = options[:retry_policy].to_grpc if options[:retry_policy]
|
524
511
|
grpc = service.create_subscription name, subscription_name, options
|
525
512
|
Subscription.from_grpc grpc, service
|
526
513
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: google-cloud-pubsub
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.11.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mike Moore
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2022-
|
12
|
+
date: 2022-08-01 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: concurrent-ruby
|
@@ -53,6 +53,20 @@ dependencies:
|
|
53
53
|
- - "~>"
|
54
54
|
- !ruby/object:Gem::Version
|
55
55
|
version: '0.0'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: retriable
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - "~>"
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '3.1'
|
63
|
+
type: :runtime
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - "~>"
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '3.1'
|
56
70
|
- !ruby/object:Gem::Dependency
|
57
71
|
name: autotest-suffix
|
58
72
|
requirement: !ruby/object:Gem::Requirement
|
@@ -101,14 +115,14 @@ dependencies:
|
|
101
115
|
requirements:
|
102
116
|
- - "~>"
|
103
117
|
- !ruby/object:Gem::Version
|
104
|
-
version: '5.
|
118
|
+
version: '5.16'
|
105
119
|
type: :development
|
106
120
|
prerelease: false
|
107
121
|
version_requirements: !ruby/object:Gem::Requirement
|
108
122
|
requirements:
|
109
123
|
- - "~>"
|
110
124
|
- !ruby/object:Gem::Version
|
111
|
-
version: '5.
|
125
|
+
version: '5.16'
|
112
126
|
- !ruby/object:Gem::Dependency
|
113
127
|
name: minitest-autotest
|
114
128
|
requirement: !ruby/object:Gem::Requirement
|
@@ -227,6 +241,7 @@ files:
|
|
227
241
|
- TROUBLESHOOTING.md
|
228
242
|
- lib/google-cloud-pubsub.rb
|
229
243
|
- lib/google/cloud/pubsub.rb
|
244
|
+
- lib/google/cloud/pubsub/acknowledge_result.rb
|
230
245
|
- lib/google/cloud/pubsub/async_publisher.rb
|
231
246
|
- lib/google/cloud/pubsub/async_publisher/batch.rb
|
232
247
|
- lib/google/cloud/pubsub/batch_publisher.rb
|