shoryuken 7.0.2 → 7.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/push.yml +3 -3
- data/.github/workflows/specs.yml +32 -5
- data/.github/workflows/verify-action-pins.yml +1 -1
- data/.ruby-version +1 -1
- data/CHANGELOG.md +107 -0
- data/bin/cli/sqs.rb +7 -0
- data/bin/integrations +52 -34
- data/lib/active_job/queue_adapters/shoryuken_adapter.rb +3 -1
- data/lib/shoryuken/active_job/current_attributes.rb +35 -7
- data/lib/shoryuken/fetcher.rb +7 -1
- data/lib/shoryuken/helpers/timer_task.rb +19 -2
- data/lib/shoryuken/launcher.rb +21 -5
- data/lib/shoryuken/manager.rb +38 -5
- data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +32 -2
- data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +8 -3
- data/lib/shoryuken/middleware/server/non_retryable_exception.rb +17 -8
- data/lib/shoryuken/options.rb +12 -1
- data/lib/shoryuken/polling/strict_priority.rb +26 -14
- data/lib/shoryuken/polling/weighted_round_robin.rb +41 -27
- data/lib/shoryuken/queue.rb +8 -1
- data/lib/shoryuken/util.rb +4 -1
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker.rb +5 -1
- data/lib/shoryuken.rb +2 -0
- data/renovate.json +16 -2
- data/spec/integration/active_job/current_attributes/cross_job_reset_spec.rb +47 -0
- data/spec/integration/active_job/current_attributes/incremental_persist_spec.rb +76 -0
- data/spec/integration/active_job/fifo_dedup_opt_out/fifo_dedup_opt_out_spec.rb +67 -0
- data/spec/integration/auto_extend_visibility/short_visibility_timeout_spec.rb +52 -0
- data/spec/integration/concurrent_processing/processor_accounting_spec.rb +94 -0
- data/spec/integration/fifo_ordering/fifo_max_messages_cap_spec.rb +96 -0
- data/spec/integration/launcher/double_graceful_stop_spec.rb +71 -0
- data/spec/integration/launcher/embedded_dispatch_error_spec.rb +85 -0
- data/spec/integration/launcher/global_executor_preserved_spec.rb +76 -0
- data/spec/integration/launcher/graceful_stop_timeout_spec.rb +74 -0
- data/spec/integration/message_operations/partial_batch_delete_spec.rb +67 -0
- data/spec/integration/non_retryable_exception/non_retryable_exception_spec.rb +1 -1
- data/spec/integration/non_retryable_exception/with_retry_intervals_spec.rb +115 -0
- data/spec/integrations_helper.rb +10 -9
- data/spec/lib/shoryuken/fetcher_spec.rb +13 -0
- data/spec/lib/shoryuken/helpers/timer_task_spec.rb +24 -0
- data/spec/lib/shoryuken/launcher_spec.rb +38 -0
- data/spec/lib/shoryuken/manager_spec.rb +99 -0
- data/spec/lib/shoryuken/middleware/server/auto_extend_visibility_spec.rb +35 -0
- data/spec/lib/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +56 -0
- data/spec/lib/shoryuken/polling/strict_priority_spec.rb +25 -0
- data/spec/lib/shoryuken/polling/weighted_round_robin_spec.rb +50 -0
- data/spec/lib/shoryuken/queue_spec.rb +37 -0
- data/spec/lib/shoryuken/util_spec.rb +26 -0
- data/spec/shared_examples_for_active_job.rb +18 -0
- data/spec/spec_helper.rb +26 -7
- metadata +26 -2
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# This spec tests the interplay between non_retryable_exceptions and
|
|
4
|
+
# retry_intervals (exponential backoff) when both are configured on the
|
|
5
|
+
# same worker.
|
|
6
|
+
#
|
|
7
|
+
# Expected behavior: non_retryable_exceptions takes precedence - a message
|
|
8
|
+
# raising a non-retryable exception must be deleted immediately and never
|
|
9
|
+
# retried, while other exceptions still go through the backoff schedule.
|
|
10
|
+
#
|
|
11
|
+
# Regression: ExponentialBackoffRetry sits inside NonRetryableException in
|
|
12
|
+
# the default middleware chain and used to swallow every exception after
|
|
13
|
+
# scheduling a retry, so NonRetryableException never saw non-retryable
|
|
14
|
+
# errors and the message was retried (with the last interval repeated)
|
|
15
|
+
# instead of being deleted.
|
|
16
|
+
|
|
17
|
+
require 'timeout'
|
|
18
|
+
|
|
19
|
+
setup_sqs
|
|
20
|
+
|
|
21
|
+
DT.clear
|
|
22
|
+
|
|
23
|
+
queue_name = DT.queue
|
|
24
|
+
create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '2' })
|
|
25
|
+
Shoryuken.add_group('default', 1)
|
|
26
|
+
Shoryuken.add_queue(queue_name, 1, 'default')
|
|
27
|
+
|
|
28
|
+
# Exception classes for testing
|
|
29
|
+
PermanentValidationError = Class.new(StandardError)
|
|
30
|
+
TransientNetworkError = Class.new(StandardError)
|
|
31
|
+
|
|
32
|
+
# Worker with BOTH retry_intervals and non_retryable_exceptions configured
|
|
33
|
+
combined_worker = Class.new do
|
|
34
|
+
include Shoryuken::Worker
|
|
35
|
+
|
|
36
|
+
shoryuken_options auto_delete: true,
|
|
37
|
+
batch: false,
|
|
38
|
+
retry_intervals: [1, 1],
|
|
39
|
+
non_retryable_exceptions: [PermanentValidationError]
|
|
40
|
+
|
|
41
|
+
def perform(sqs_msg, body)
|
|
42
|
+
receive_count = sqs_msg.attributes['ApproximateReceiveCount'].to_i
|
|
43
|
+
DT[:attempts] << { receive_count: receive_count, body: body, time: Time.now }
|
|
44
|
+
|
|
45
|
+
case body
|
|
46
|
+
when 'non_retryable'
|
|
47
|
+
raise PermanentValidationError, 'Permanently invalid input'
|
|
48
|
+
when 'retryable'
|
|
49
|
+
# Fail on first 2 attempts (covered by retry_intervals), succeed on 3rd
|
|
50
|
+
raise TransientNetworkError, "Temporary failure on attempt #{receive_count}" if receive_count < 3
|
|
51
|
+
|
|
52
|
+
DT[:successful_processing] << { body: body, final_receive_count: receive_count }
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
combined_worker.get_shoryuken_options['queue'] = queue_name
|
|
58
|
+
Shoryuken.register_worker(queue_name, combined_worker)
|
|
59
|
+
|
|
60
|
+
queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
|
|
61
|
+
|
|
62
|
+
launcher = Shoryuken::Launcher.new
|
|
63
|
+
launcher.start
|
|
64
|
+
|
|
65
|
+
begin
|
|
66
|
+
# Test 1: a non-retryable exception must win over the backoff schedule -
|
|
67
|
+
# the message is attempted exactly once and deleted, never retried.
|
|
68
|
+
Shoryuken::Client.sqs.send_message(
|
|
69
|
+
queue_url: queue_url,
|
|
70
|
+
message_body: 'non_retryable'
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
Timeout.timeout(10) { sleep 0.5 until DT[:attempts].size >= 1 }
|
|
74
|
+
|
|
75
|
+
# Give a buggy retry every chance to happen: with retry_intervals [1, 1]
|
|
76
|
+
# a swallowed exception would bring the message back after ~1s.
|
|
77
|
+
sleep 5
|
|
78
|
+
|
|
79
|
+
non_retryable_attempts = DT[:attempts].select { |a| a[:body] == 'non_retryable' }
|
|
80
|
+
assert_equal(
|
|
81
|
+
1,
|
|
82
|
+
non_retryable_attempts.size,
|
|
83
|
+
'Non-retryable exception must not be retried even when retry_intervals is configured ' \
|
|
84
|
+
"(got #{non_retryable_attempts.size} attempts)"
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
attributes = Shoryuken::Client.sqs.get_queue_attributes(
|
|
88
|
+
queue_url: queue_url,
|
|
89
|
+
attribute_names: %w[ApproximateNumberOfMessages ApproximateNumberOfMessagesNotVisible]
|
|
90
|
+
).attributes
|
|
91
|
+
|
|
92
|
+
total_messages = attributes['ApproximateNumberOfMessages'].to_i +
|
|
93
|
+
attributes['ApproximateNumberOfMessagesNotVisible'].to_i
|
|
94
|
+
assert_equal(
|
|
95
|
+
0,
|
|
96
|
+
total_messages,
|
|
97
|
+
'Message with non-retryable exception must be deleted immediately, not scheduled for retry'
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
# Test 2: exceptions NOT in the non-retryable list still follow the
|
|
101
|
+
# backoff schedule and eventually succeed.
|
|
102
|
+
Shoryuken::Client.sqs.send_message(
|
|
103
|
+
queue_url: queue_url,
|
|
104
|
+
message_body: 'retryable'
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
Timeout.timeout(20) { sleep 0.5 until DT[:successful_processing].size >= 1 }
|
|
108
|
+
|
|
109
|
+
retryable_attempts = DT[:attempts].select { |a| a[:body] == 'retryable' }
|
|
110
|
+
assert(retryable_attempts.size >= 3, 'Retryable exception should still go through the backoff schedule')
|
|
111
|
+
assert_equal('retryable', DT[:successful_processing].first[:body])
|
|
112
|
+
assert_equal(3, DT[:successful_processing].first[:final_receive_count])
|
|
113
|
+
ensure
|
|
114
|
+
Timeout.timeout(30) { launcher.stop }
|
|
115
|
+
end
|
data/spec/integrations_helper.rb
CHANGED
|
@@ -2,12 +2,15 @@
|
|
|
2
2
|
|
|
3
3
|
# Integration test helper for process-isolated testing
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
Warning[:deprecated] = true
|
|
5
|
+
require 'warning'
|
|
6
|
+
|
|
8
7
|
$VERBOSE = true
|
|
9
8
|
|
|
10
|
-
|
|
9
|
+
if Warning.respond_to?(:categories)
|
|
10
|
+
(Warning.categories - %i[experimental]).each do |cat|
|
|
11
|
+
Warning[cat] = true
|
|
12
|
+
end
|
|
13
|
+
end
|
|
11
14
|
|
|
12
15
|
# Process warnings and raise on unexpected ones from our code
|
|
13
16
|
Warning.process do |warning|
|
|
@@ -17,10 +20,6 @@ Warning.process do |warning|
|
|
|
17
20
|
# Filter out warnings we don't care about in specs
|
|
18
21
|
next if warning.include?('_spec')
|
|
19
22
|
|
|
20
|
-
# We redefine methods to simulate various scenarios in tests
|
|
21
|
-
next if warning.include?('previous definition of')
|
|
22
|
-
next if warning.include?('method redefined')
|
|
23
|
-
|
|
24
23
|
# Ignore vendor directory
|
|
25
24
|
next if warning.include?('vendor/')
|
|
26
25
|
|
|
@@ -191,7 +190,9 @@ module IntegrationsHelper
|
|
|
191
190
|
launcher.start
|
|
192
191
|
Timeout.timeout(timeout) { sleep 0.5 until yield }
|
|
193
192
|
ensure
|
|
194
|
-
|
|
193
|
+
# Guard stop with a hard deadline so a stuck dispatch thread (e.g. an SQS
|
|
194
|
+
# HTTP call that never returns) can't block the spec process forever.
|
|
195
|
+
Timeout.timeout(30) { launcher.stop }
|
|
195
196
|
end
|
|
196
197
|
|
|
197
198
|
# Simple mock object
|
|
@@ -113,6 +113,19 @@ RSpec.describe Shoryuken::Fetcher do
|
|
|
113
113
|
subject.fetch(queue_config, limit)
|
|
114
114
|
end
|
|
115
115
|
|
|
116
|
+
it 'caps at one message even when the queue config requests more' do
|
|
117
|
+
# A custom polling strategy can put max_number_of_messages in the
|
|
118
|
+
# QueueConfiguration options; the FIFO one-at-a-time guard must still win
|
|
119
|
+
# so SQS can't return several messages from the same group.
|
|
120
|
+
config = Shoryuken::Polling::QueueConfiguration.new(queue_name, max_number_of_messages: 10)
|
|
121
|
+
|
|
122
|
+
allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
|
123
|
+
expect(queue).to receive(:receive_messages)
|
|
124
|
+
.with(hash_including(max_number_of_messages: 1)).and_return([])
|
|
125
|
+
|
|
126
|
+
subject.fetch(config, limit)
|
|
127
|
+
end
|
|
128
|
+
|
|
116
129
|
context 'with batch=true' do
|
|
117
130
|
it 'polls the provided limit' do
|
|
118
131
|
# see https://github.com/ruby-shoryuken/shoryuken/pull/530
|
|
@@ -295,4 +295,28 @@ RSpec.describe Shoryuken::Helpers::TimerTask do
|
|
|
295
295
|
expect(false_count).to eq(4)
|
|
296
296
|
end
|
|
297
297
|
end
|
|
298
|
+
|
|
299
|
+
describe 'thread priority' do
|
|
300
|
+
it 'does not inherit a lowered priority from the creating thread' do
|
|
301
|
+
observed = ::Queue.new
|
|
302
|
+
|
|
303
|
+
# Shoryuken starts the visibility-extension timer from a worker thread
|
|
304
|
+
# whose priority is lowered to Shoryuken.thread_priority (default -1).
|
|
305
|
+
# The timer thread must not inherit that lowered priority, otherwise a
|
|
306
|
+
# delayed extension under load can miss the visibility timeout and let the
|
|
307
|
+
# message be redelivered (double processed).
|
|
308
|
+
creator = Thread.new do
|
|
309
|
+
Thread.current.priority = -1
|
|
310
|
+
timer = described_class.new(execution_interval: 0.05) do
|
|
311
|
+
observed << Thread.current.priority
|
|
312
|
+
end
|
|
313
|
+
timer.execute
|
|
314
|
+
sleep(0.15)
|
|
315
|
+
timer.kill
|
|
316
|
+
end
|
|
317
|
+
creator.join
|
|
318
|
+
|
|
319
|
+
expect(observed.pop).to eq(0)
|
|
320
|
+
end
|
|
321
|
+
end
|
|
298
322
|
end
|
|
@@ -78,6 +78,22 @@ RSpec.describe Shoryuken::Launcher do
|
|
|
78
78
|
expect(first_group_manager).to have_received(:stop_new_dispatching)
|
|
79
79
|
expect(second_group_manager).to have_received(:stop_new_dispatching)
|
|
80
80
|
end
|
|
81
|
+
|
|
82
|
+
it 'bounds the wait for termination by the configured timeout' do
|
|
83
|
+
allow(executor).to receive(:shutdown)
|
|
84
|
+
expect(executor).to receive(:wait_for_termination).with(Shoryuken.options[:timeout]).and_return(true)
|
|
85
|
+
expect(executor).not_to receive(:kill)
|
|
86
|
+
|
|
87
|
+
subject.stop
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
it 'kills the executor when workers do not finish within the timeout' do
|
|
91
|
+
allow(executor).to receive(:shutdown)
|
|
92
|
+
allow(executor).to receive(:wait_for_termination).and_return(false)
|
|
93
|
+
expect(executor).to receive(:kill)
|
|
94
|
+
|
|
95
|
+
subject.stop
|
|
96
|
+
end
|
|
81
97
|
end
|
|
82
98
|
|
|
83
99
|
describe '#stop!' do
|
|
@@ -102,6 +118,28 @@ RSpec.describe Shoryuken::Launcher do
|
|
|
102
118
|
end
|
|
103
119
|
end
|
|
104
120
|
|
|
121
|
+
describe 'executor ownership' do
|
|
122
|
+
context 'when no launcher_executor is configured' do
|
|
123
|
+
before { allow(Shoryuken).to receive(:launcher_executor).and_return(nil) }
|
|
124
|
+
|
|
125
|
+
it 'uses a dedicated executor rather than the process-global IO executor' do
|
|
126
|
+
expect(subject.send(:executor)).not_to be(Concurrent.global_io_executor)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
it 'does not shut down or kill the global IO executor on stop!' do
|
|
130
|
+
allow(first_group_manager).to receive(:stop_new_dispatching)
|
|
131
|
+
allow(second_group_manager).to receive(:stop_new_dispatching)
|
|
132
|
+
|
|
133
|
+
subject.stop!
|
|
134
|
+
|
|
135
|
+
# Verify by inspecting state after the call rather than mocking methods
|
|
136
|
+
# on the process-global singleton: RSpec proxies on a singleton used by
|
|
137
|
+
# background concurrent-ruby threads can deadlock on Ruby 3.2.
|
|
138
|
+
expect(Concurrent.global_io_executor).to be_running
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
105
143
|
describe '#stopping?' do
|
|
106
144
|
it 'returns false by default' do
|
|
107
145
|
expect(subject.stopping?).to be false
|
|
@@ -34,6 +34,39 @@ RSpec.describe Shoryuken::Manager do
|
|
|
34
34
|
end
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
+
describe '#await_dispatching_in_progress' do
|
|
38
|
+
it 'returns for repeated graceful stops (e.g. TSTP followed by USR1)' do
|
|
39
|
+
subject.stop_new_dispatching
|
|
40
|
+
# The dispatch loop observes the stop flag and releases all waiters
|
|
41
|
+
subject.start
|
|
42
|
+
|
|
43
|
+
waiter = Thread.new do
|
|
44
|
+
subject.await_dispatching_in_progress
|
|
45
|
+
subject.await_dispatching_in_progress
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
expect(waiter.join(5)).to eq(waiter), 'await_dispatching_in_progress deadlocked on a repeated stop'
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
context 'when the executor rejects the dispatch post' do
|
|
52
|
+
let(:executor) do
|
|
53
|
+
double('executor', running?: true).tap do |rejecting_executor|
|
|
54
|
+
allow(rejecting_executor).to receive(:post).and_raise(Concurrent::RejectedExecutionError)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
it 'releases waiters instead of leaving the signal queue unsignaled' do
|
|
59
|
+
# A hard stop can shut the executor down between the running? check and
|
|
60
|
+
# the post; the dispatch chain must still release stop waiters
|
|
61
|
+
subject.start
|
|
62
|
+
|
|
63
|
+
waiter = Thread.new { subject.await_dispatching_in_progress }
|
|
64
|
+
|
|
65
|
+
expect(waiter.join(5)).to eq(waiter), 'await_dispatching_in_progress deadlocked after a rejected post'
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
37
70
|
describe '#start' do
|
|
38
71
|
before do
|
|
39
72
|
# prevent dispatch loop
|
|
@@ -153,6 +186,44 @@ RSpec.describe Shoryuken::Manager do
|
|
|
153
186
|
end
|
|
154
187
|
end
|
|
155
188
|
|
|
189
|
+
describe '#assign' do
|
|
190
|
+
let(:sqs_msg) { double(Shoryuken::Message, message_id: SecureRandom.uuid) }
|
|
191
|
+
let(:sqs_queue) { double(Shoryuken::Queue, fifo?: false) }
|
|
192
|
+
|
|
193
|
+
before do
|
|
194
|
+
allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
|
|
195
|
+
allow(Shoryuken::Processor).to receive(:process)
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
it 'runs completion exactly once on success' do
|
|
199
|
+
expect(subject).to receive(:processor_done).with(queue).once.and_call_original
|
|
200
|
+
|
|
201
|
+
subject.send(:assign, queue, sqs_msg)
|
|
202
|
+
|
|
203
|
+
expect(subject.send(:busy)).to eq(0)
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
it 'runs completion exactly once when processing raises' do
|
|
207
|
+
allow(Shoryuken::Processor).to receive(:process).and_raise('processing failed')
|
|
208
|
+
|
|
209
|
+
expect(subject).to receive(:processor_done).with(queue).once.and_call_original
|
|
210
|
+
|
|
211
|
+
subject.send(:assign, queue, sqs_msg)
|
|
212
|
+
|
|
213
|
+
expect(subject.send(:busy)).to eq(0)
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
it 'does not double decrement the busy counter when completion raises' do
|
|
217
|
+
allow(sqs_queue).to receive(:fifo?).and_return(true)
|
|
218
|
+
allow(polling_strategy).to receive(:message_processed).and_raise('callback failed')
|
|
219
|
+
|
|
220
|
+
subject.send(:assign, queue, sqs_msg)
|
|
221
|
+
|
|
222
|
+
expect(polling_strategy).to have_received(:message_processed).once
|
|
223
|
+
expect(subject.send(:busy)).to eq(0)
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
|
|
156
227
|
describe '#processor_done' do
|
|
157
228
|
let(:sqs_queue) { double Shoryuken::Queue }
|
|
158
229
|
|
|
@@ -177,6 +248,34 @@ RSpec.describe Shoryuken::Manager do
|
|
|
177
248
|
end
|
|
178
249
|
end
|
|
179
250
|
|
|
251
|
+
describe '#handle_dispatch_error' do
|
|
252
|
+
let(:error) { StandardError.new('boom') }
|
|
253
|
+
|
|
254
|
+
context 'when running under the CLI (server mode)' do
|
|
255
|
+
before { allow(Shoryuken).to receive(:server?).and_return(true) }
|
|
256
|
+
|
|
257
|
+
it 'stops the manager and signals the process to shut down' do
|
|
258
|
+
expect(Process).to receive(:kill).with('USR1', Process.pid)
|
|
259
|
+
|
|
260
|
+
subject.send(:handle_dispatch_error, error)
|
|
261
|
+
|
|
262
|
+
expect(subject.running?).to be false
|
|
263
|
+
end
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
context 'when embedded (no Runner)' do
|
|
267
|
+
before { allow(Shoryuken).to receive(:server?).and_return(false) }
|
|
268
|
+
|
|
269
|
+
it 'stops the manager without signalling the process' do
|
|
270
|
+
expect(Process).not_to receive(:kill)
|
|
271
|
+
|
|
272
|
+
subject.send(:handle_dispatch_error, error)
|
|
273
|
+
|
|
274
|
+
expect(subject.running?).to be false
|
|
275
|
+
end
|
|
276
|
+
end
|
|
277
|
+
end
|
|
278
|
+
|
|
180
279
|
describe '#running?' do
|
|
181
280
|
context 'when the executor is running' do
|
|
182
281
|
before do
|
|
@@ -114,4 +114,39 @@ RSpec.describe Shoryuken::Middleware::Server::AutoExtendVisibility do
|
|
|
114
114
|
Runner.new.run_and_sleep(TestWorker.new, queue, sqs_msg, visibility_timeout)
|
|
115
115
|
end
|
|
116
116
|
end
|
|
117
|
+
|
|
118
|
+
context 'when the visibility timeout is too short for the upfront margin' do
|
|
119
|
+
# interval would be visibility_timeout - extend_upfront == 0, which used to
|
|
120
|
+
# make TimerTask raise before the worker ran.
|
|
121
|
+
let(:visibility_timeout) { extend_upfront }
|
|
122
|
+
|
|
123
|
+
it 'runs the worker instead of raising' do
|
|
124
|
+
TestWorker.get_shoryuken_options['auto_visibility_timeout'] = true
|
|
125
|
+
allow(sqs_msg).to receive(:queue) { sqs_queue }
|
|
126
|
+
allow(sqs_msg).to receive(:message_id).and_return('test-message-id')
|
|
127
|
+
allow(sqs_msg).to receive(:change_visibility)
|
|
128
|
+
|
|
129
|
+
ran = false
|
|
130
|
+
subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { ran = true }
|
|
131
|
+
|
|
132
|
+
expect(ran).to be true
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
context 'when the visibility timeout is zero' do
|
|
137
|
+
let(:visibility_timeout) { 0 }
|
|
138
|
+
|
|
139
|
+
it 'warns and runs the worker without scheduling an extension' do
|
|
140
|
+
TestWorker.get_shoryuken_options['auto_visibility_timeout'] = true
|
|
141
|
+
allow(sqs_msg).to receive(:queue) { sqs_queue }
|
|
142
|
+
allow(sqs_msg).to receive(:message_id).and_return('test-message-id')
|
|
143
|
+
expect(sqs_msg).not_to receive(:change_visibility)
|
|
144
|
+
expect(Shoryuken.logger).to receive(:warn)
|
|
145
|
+
|
|
146
|
+
ran = false
|
|
147
|
+
subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { ran = true }
|
|
148
|
+
|
|
149
|
+
expect(ran).to be true
|
|
150
|
+
end
|
|
151
|
+
end
|
|
117
152
|
end
|
|
@@ -92,6 +92,62 @@ RSpec.describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
|
|
|
92
92
|
|
|
93
93
|
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
|
|
94
94
|
end
|
|
95
|
+
|
|
96
|
+
it 'keeps reusing the last interval for far-later attempts (does not give up)' do
|
|
97
|
+
TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
|
|
98
|
+
|
|
99
|
+
allow(sqs_msg).to receive(:attributes) { { 'ApproximateReceiveCount' => 10 } }
|
|
100
|
+
allow(sqs_msg).to receive(:queue) { sqs_queue }
|
|
101
|
+
expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
|
|
102
|
+
|
|
103
|
+
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
context 'and the exception is non-retryable' do
|
|
108
|
+
it 're-raises so NonRetryableException can delete the message' do
|
|
109
|
+
TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
|
|
110
|
+
TestWorker.get_shoryuken_options['non_retryable_exceptions'] = [ArgumentError]
|
|
111
|
+
|
|
112
|
+
expect(sqs_msg).not_to receive(:change_visibility)
|
|
113
|
+
|
|
114
|
+
expect {
|
|
115
|
+
subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise ArgumentError, 'permanently invalid' }
|
|
116
|
+
}.to raise_error(ArgumentError, 'permanently invalid')
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
it 're-raises when a non_retryable_exceptions lambda returns true' do
|
|
120
|
+
TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
|
|
121
|
+
TestWorker.get_shoryuken_options['non_retryable_exceptions'] = ->(e) { e.message.include?('permanent') }
|
|
122
|
+
|
|
123
|
+
expect(sqs_msg).not_to receive(:change_visibility)
|
|
124
|
+
|
|
125
|
+
expect {
|
|
126
|
+
subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'permanent failure' }
|
|
127
|
+
}.to raise_error(RuntimeError, 'permanent failure')
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
it 'still retries when a non_retryable_exceptions lambda returns false' do
|
|
131
|
+
TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
|
|
132
|
+
TestWorker.get_shoryuken_options['non_retryable_exceptions'] = ->(e) { e.message.include?('permanent') }
|
|
133
|
+
|
|
134
|
+
allow(sqs_msg).to receive(:queue) { sqs_queue }
|
|
135
|
+
expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 300)
|
|
136
|
+
|
|
137
|
+
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'transient failure' } }
|
|
138
|
+
.not_to raise_error
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
it 'still retries exceptions not in the non-retryable list' do
|
|
142
|
+
TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
|
|
143
|
+
TestWorker.get_shoryuken_options['non_retryable_exceptions'] = [ArgumentError]
|
|
144
|
+
|
|
145
|
+
allow(sqs_msg).to receive(:queue) { sqs_queue }
|
|
146
|
+
expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 300)
|
|
147
|
+
|
|
148
|
+
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'transient failure' } }
|
|
149
|
+
.not_to raise_error
|
|
150
|
+
end
|
|
95
151
|
end
|
|
96
152
|
|
|
97
153
|
it 'limits the visibility timeout to 12 hours' do
|
|
@@ -155,4 +155,29 @@ RSpec.describe Shoryuken::Polling::StrictPriority do
|
|
|
155
155
|
expect(strategy.active_queues).to eq([[queue1, 2], [queue2, 1]])
|
|
156
156
|
end
|
|
157
157
|
end
|
|
158
|
+
|
|
159
|
+
describe 'thread safety' do
|
|
160
|
+
it 'serializes concurrent next_queue, messages_found and message_processed' do
|
|
161
|
+
strategy = Shoryuken::Polling::StrictPriority.new([queue1, queue2], 10)
|
|
162
|
+
|
|
163
|
+
errors = []
|
|
164
|
+
errors_mutex = Mutex.new
|
|
165
|
+
|
|
166
|
+
threads = Array.new(6) do
|
|
167
|
+
Thread.new do
|
|
168
|
+
50.times do |i|
|
|
169
|
+
strategy.next_queue
|
|
170
|
+
strategy.messages_found(queue1, i.even? ? 0 : 1)
|
|
171
|
+
strategy.message_processed(queue1)
|
|
172
|
+
strategy.active_queues
|
|
173
|
+
end
|
|
174
|
+
rescue => e
|
|
175
|
+
errors_mutex.synchronize { errors << e }
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
threads.each(&:join)
|
|
179
|
+
|
|
180
|
+
expect(errors).to be_empty
|
|
181
|
+
end
|
|
182
|
+
end
|
|
158
183
|
end
|
|
@@ -138,5 +138,55 @@ RSpec.describe Shoryuken::Polling::WeightedRoundRobin do
|
|
|
138
138
|
expect(subject.next_queue).to eq(queue1) # implicitly unpauses queue2
|
|
139
139
|
expect(subject.active_queues).to eq([[queue1, 2], [queue2, 1]])
|
|
140
140
|
end
|
|
141
|
+
|
|
142
|
+
it 'unpauses a processed queue stuck behind an earlier-paused queue' do
|
|
143
|
+
queues << queue1
|
|
144
|
+
queues << queue2
|
|
145
|
+
|
|
146
|
+
allow(subject).to receive(:delay).and_return(10)
|
|
147
|
+
now = Time.now
|
|
148
|
+
allow(Time).to receive(:now).and_return(now)
|
|
149
|
+
|
|
150
|
+
subject.messages_found(queue1, 0) # queue1 paused until now + 10
|
|
151
|
+
|
|
152
|
+
now += 1
|
|
153
|
+
allow(Time).to receive(:now).and_return(now)
|
|
154
|
+
subject.messages_found(queue2, 0) # queue2 paused until now + 10 (after queue1)
|
|
155
|
+
|
|
156
|
+
# both queues paused, nothing pollable yet
|
|
157
|
+
expect(subject.next_queue).to eq(nil)
|
|
158
|
+
|
|
159
|
+
# queue2 finishes processing and is marked ready while queue1 is still paused
|
|
160
|
+
subject.message_processed(queue2)
|
|
161
|
+
|
|
162
|
+
# queue2 must be pollable even though the earlier-paused queue1 is still paused
|
|
163
|
+
expect(subject.next_queue).to eq(queue2)
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
describe 'thread safety' do
|
|
168
|
+
it 'serializes concurrent next_queue, messages_found and message_processed' do
|
|
169
|
+
queues << queue1
|
|
170
|
+
queues << queue2
|
|
171
|
+
|
|
172
|
+
errors = []
|
|
173
|
+
errors_mutex = Mutex.new
|
|
174
|
+
|
|
175
|
+
threads = Array.new(6) do
|
|
176
|
+
Thread.new do
|
|
177
|
+
50.times do |i|
|
|
178
|
+
subject.next_queue
|
|
179
|
+
subject.messages_found(queue1, i.even? ? 0 : 1)
|
|
180
|
+
subject.message_processed(queue1)
|
|
181
|
+
subject.active_queues
|
|
182
|
+
end
|
|
183
|
+
rescue => e
|
|
184
|
+
errors_mutex.synchronize { errors << e }
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
threads.each(&:join)
|
|
188
|
+
|
|
189
|
+
expect(errors).to be_empty
|
|
190
|
+
end
|
|
141
191
|
end
|
|
142
192
|
end
|
|
@@ -99,6 +99,43 @@ RSpec.describe Shoryuken::Queue do
|
|
|
99
99
|
|
|
100
100
|
subject.delete_messages(entries: entries)
|
|
101
101
|
end
|
|
102
|
+
|
|
103
|
+
it 'returns true' do
|
|
104
|
+
failure = double(id: 'id', code: 'code', message: '...', sender_fault: false)
|
|
105
|
+
allow(sqs).to receive(:delete_message_batch).and_return(double(failed: [failure]))
|
|
106
|
+
allow(subject).to receive(:logger).and_return(double('Logger', error: nil))
|
|
107
|
+
|
|
108
|
+
expect(subject.delete_messages(entries: entries)).to be true
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
context 'when several deletes fail' do
|
|
113
|
+
let(:failures) do
|
|
114
|
+
[
|
|
115
|
+
double(id: '1', code: 'c1', message: 'm1', sender_fault: false),
|
|
116
|
+
double(id: '2', code: 'c2', message: 'm2', sender_fault: true)
|
|
117
|
+
]
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
it 'logs every failure, not just the first' do
|
|
121
|
+
logger = double('Logger')
|
|
122
|
+
allow(sqs).to receive(:delete_message_batch).and_return(double(failed: failures))
|
|
123
|
+
allow(subject).to receive(:logger).and_return(logger)
|
|
124
|
+
|
|
125
|
+
# Real Logger#error returns true; that truthy return made the old `any?`
|
|
126
|
+
# short-circuit after the first failure, so the rest went unlogged.
|
|
127
|
+
expect(logger).to receive(:error).twice.and_return(true)
|
|
128
|
+
|
|
129
|
+
subject.delete_messages(entries: entries)
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
context 'when all deletes succeed' do
|
|
134
|
+
it 'returns false' do
|
|
135
|
+
allow(sqs).to receive(:delete_message_batch).and_return(double(failed: []))
|
|
136
|
+
|
|
137
|
+
expect(subject.delete_messages(entries: entries)).to be false
|
|
138
|
+
end
|
|
102
139
|
end
|
|
103
140
|
end
|
|
104
141
|
|
|
@@ -69,5 +69,31 @@ describe 'Shoryuken::Util' do
|
|
|
69
69
|
expect(value_holder).to receive(:value=).with([:with_options, { my_option: :some_option }])
|
|
70
70
|
subject.fire_event(:some_event, false, my_option: :some_option)
|
|
71
71
|
end
|
|
72
|
+
|
|
73
|
+
it 'fires handlers in reverse order consistently across repeated reverse firings' do
|
|
74
|
+
order = []
|
|
75
|
+
Shoryuken.options[:lifecycle_events][:some_event] = [
|
|
76
|
+
->(_) { order << :a },
|
|
77
|
+
->(_) { order << :b },
|
|
78
|
+
->(_) { order << :c }
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
# e.g. :shutdown is fired with reverse: true and can fire more than once
|
|
82
|
+
# (stop then stop!); every firing must use the same reversed order.
|
|
83
|
+
subject.fire_event(:some_event, true)
|
|
84
|
+
subject.fire_event(:some_event, true)
|
|
85
|
+
|
|
86
|
+
expect(order).to eq(%i[c b a c b a])
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
it 'does not mutate the stored handler array when firing in reverse' do
|
|
90
|
+
handlers = [->(_) {}, ->(_) {}]
|
|
91
|
+
original = handlers.dup
|
|
92
|
+
Shoryuken.options[:lifecycle_events][:some_event] = handlers
|
|
93
|
+
|
|
94
|
+
subject.fire_event(:some_event, true)
|
|
95
|
+
|
|
96
|
+
expect(Shoryuken.options[:lifecycle_events][:some_event]).to eq(original)
|
|
97
|
+
end
|
|
72
98
|
end
|
|
73
99
|
end
|
|
@@ -55,6 +55,24 @@ RSpec.shared_examples 'active_job_adapters' do
|
|
|
55
55
|
subject.enqueue(job)
|
|
56
56
|
end
|
|
57
57
|
|
|
58
|
+
context 'when active_job_fifo_message_deduplication is disabled' do
|
|
59
|
+
before { Shoryuken.active_job_fifo_message_deduplication = false }
|
|
60
|
+
|
|
61
|
+
it 'does not set an auto-generated message_deduplication_id' do
|
|
62
|
+
expect(queue).to receive(:send_message) do |hash|
|
|
63
|
+
expect(hash).not_to have_key(:message_deduplication_id)
|
|
64
|
+
end
|
|
65
|
+
subject.enqueue(job)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
it 'still honors an explicit message_deduplication_id' do
|
|
69
|
+
expect(queue).to receive(:send_message) do |hash|
|
|
70
|
+
expect(hash[:message_deduplication_id]).to eq('explicit-id')
|
|
71
|
+
end
|
|
72
|
+
subject.enqueue(job, message_deduplication_id: 'explicit-id')
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
58
76
|
context 'with message_deduplication_id' do
|
|
59
77
|
context 'when message_deduplication_id is specified in options' do
|
|
60
78
|
it 'should enqueue a message with the deduplication_id specified in options' do
|