shoryuken 7.0.3 → 7.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/push.yml +3 -3
- data/.github/workflows/specs.yml +4 -4
- data/.github/workflows/verify-action-pins.yml +1 -1
- data/.ruby-version +1 -1
- data/.yard-lint.yml +29 -1
- data/CHANGELOG.md +65 -1
- data/Gemfile.lint.lock +8 -7
- data/bin/cli/sqs.rb +60 -3
- data/lib/active_job/extensions.rb +5 -8
- data/lib/active_job/queue_adapters/shoryuken_adapter.rb +3 -4
- data/lib/shoryuken/active_job/current_attributes.rb +4 -6
- data/lib/shoryuken/body_parser.rb +1 -2
- data/lib/shoryuken/client.rb +15 -6
- data/lib/shoryuken/default_exception_handler.rb +2 -3
- data/lib/shoryuken/errors.rb +2 -4
- data/lib/shoryuken/helpers/atomic_counter.rb +6 -9
- data/lib/shoryuken/helpers/atomic_hash.rb +9 -15
- data/lib/shoryuken/helpers/hash_utils.rb +5 -7
- data/lib/shoryuken/helpers/string_utils.rb +6 -8
- data/lib/shoryuken/inline_message.rb +3 -4
- data/lib/shoryuken/launcher.rb +8 -13
- data/lib/shoryuken/manager.rb +27 -1
- data/lib/shoryuken/message.rb +2 -3
- data/lib/shoryuken/middleware/chain.rb +7 -13
- data/lib/shoryuken/middleware/entry.rb +1 -2
- data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +3 -5
- data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +38 -21
- data/lib/shoryuken/middleware/server/timing.rb +2 -3
- data/lib/shoryuken/options.rb +28 -6
- data/lib/shoryuken/polling/base_strategy.rb +16 -23
- data/lib/shoryuken/polling/queue_configuration.rb +8 -12
- data/lib/shoryuken/polling/weighted_round_robin.rb +3 -4
- data/lib/shoryuken/queue.rb +33 -3
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker/default_executor.rb +11 -5
- data/lib/shoryuken/worker/inline_executor.rb +10 -5
- data/lib/shoryuken/worker.rb +10 -12
- data/lib/shoryuken.rb +4 -3
- data/shoryuken.gemspec +9 -0
- data/spec/integration/cli/find_all_spec.rb +120 -0
- data/spec/lib/shoryuken/client_spec.rb +62 -0
- data/spec/lib/shoryuken/manager_spec.rb +48 -0
- data/spec/lib/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +32 -0
- data/spec/lib/shoryuken/queue_spec.rb +86 -0
- data/spec/lib/shoryuken/worker/default_executor_spec.rb +13 -0
- data/spec/lib/shoryuken/worker/inline_executor_spec.rb +12 -0
- data/spec/spec_helper.rb +14 -13
- metadata +11 -3
data/lib/shoryuken/worker.rb
CHANGED
|
@@ -4,10 +4,9 @@ module Shoryuken
|
|
|
4
4
|
# Worker module provides the core functionality for creating Shoryuken workers
|
|
5
5
|
# that process messages from Amazon SQS queues.
|
|
6
6
|
#
|
|
7
|
-
# Including this module in a class provides methods for configuring queue processing,
|
|
8
|
-
#
|
|
9
|
-
# processing
|
|
10
|
-
# various retry and visibility timeout strategies.
|
|
7
|
+
# Including this module in a class provides methods for configuring queue processing, enqueueing jobs, and setting up
|
|
8
|
+
# middleware. Workers can be configured for different processing patterns including single message processing, batch
|
|
9
|
+
# processing, and various retry and visibility timeout strategies.
|
|
11
10
|
#
|
|
12
11
|
# @example Basic worker implementation
|
|
13
12
|
# class EmailWorker
|
|
@@ -123,7 +122,8 @@ module Shoryuken
|
|
|
123
122
|
# @option opts [Boolean] :auto_delete (false) Automatically delete messages after processing
|
|
124
123
|
# @option opts [Boolean] :auto_visibility_timeout (false) Automatically extend message visibility
|
|
125
124
|
# @option opts [Array<Integer>] :retry_intervals Exponential backoff retry intervals in seconds
|
|
126
|
-
# @option opts [Array<Class>, Proc] :non_retryable_exceptions Exception classes or lambda that should
|
|
125
|
+
# @option opts [Array<Class>, Proc] :non_retryable_exceptions Exception classes or lambda that should
|
|
126
|
+
# skip retries and delete message immediately
|
|
127
127
|
# @option opts [Hash] :sqs Additional SQS client options
|
|
128
128
|
#
|
|
129
129
|
# @example Basic worker configuration
|
|
@@ -216,9 +216,8 @@ module Shoryuken
|
|
|
216
216
|
!!get_shoryuken_options['auto_visibility_timeout']
|
|
217
217
|
end
|
|
218
218
|
|
|
219
|
-
# Checks if exponential backoff retry is configured for this worker.
|
|
220
|
-
#
|
|
221
|
-
# increasing delays between attempts.
|
|
219
|
+
# Checks if exponential backoff retry is configured for this worker. When retry intervals are specified, failed jobs
|
|
220
|
+
# will be retried with increasing delays between attempts.
|
|
222
221
|
#
|
|
223
222
|
# @return [Boolean] true if retry intervals are configured
|
|
224
223
|
#
|
|
@@ -235,10 +234,9 @@ module Shoryuken
|
|
|
235
234
|
!!get_shoryuken_options['retry_intervals']
|
|
236
235
|
end
|
|
237
236
|
|
|
238
|
-
# Checks if automatic message deletion is enabled for this worker.
|
|
239
|
-
# When
|
|
240
|
-
#
|
|
241
|
-
# or they will become visible again after the visibility timeout.
|
|
237
|
+
# Checks if automatic message deletion is enabled for this worker. When enabled, successfully processed messages are
|
|
238
|
+
# automatically deleted from the SQS queue. When disabled, you must manually delete messages or they will become
|
|
239
|
+
# visible again after the visibility timeout.
|
|
242
240
|
#
|
|
243
241
|
# @return [Boolean] true if auto delete is enabled
|
|
244
242
|
#
|
data/lib/shoryuken.rb
CHANGED
|
@@ -27,9 +27,8 @@ module Shoryuken
|
|
|
27
27
|
@_shoryuken_options ||= Shoryuken::Options.new
|
|
28
28
|
end
|
|
29
29
|
|
|
30
|
-
# Checks if the Shoryuken server is running and healthy.
|
|
31
|
-
#
|
|
32
|
-
# are running and able to process messages.
|
|
30
|
+
# Checks if the Shoryuken server is running and healthy. A server is considered healthy when all configured processing
|
|
31
|
+
# groups are running and able to process messages.
|
|
33
32
|
#
|
|
34
33
|
# @return [Boolean] true if the server is healthy
|
|
35
34
|
def self.healthy?
|
|
@@ -60,6 +59,8 @@ module Shoryuken
|
|
|
60
59
|
:active_job_queue_name_prefixing=,
|
|
61
60
|
:active_job_fifo_message_deduplication?,
|
|
62
61
|
:active_job_fifo_message_deduplication=,
|
|
62
|
+
:fifo_message_deduplication?,
|
|
63
|
+
:fifo_message_deduplication=,
|
|
63
64
|
:sqs_client,
|
|
64
65
|
:sqs_client=,
|
|
65
66
|
:sqs_client_receive_message_opts,
|
data/shoryuken.gemspec
CHANGED
|
@@ -11,6 +11,15 @@ Gem::Specification.new do |spec|
|
|
|
11
11
|
spec.homepage = 'https://github.com/ruby-shoryuken/shoryuken'
|
|
12
12
|
spec.license = 'LGPL-3.0'
|
|
13
13
|
|
|
14
|
+
spec.metadata = {
|
|
15
|
+
'homepage_uri' => spec.homepage,
|
|
16
|
+
'source_code_uri' => spec.homepage,
|
|
17
|
+
'changelog_uri' => "#{spec.homepage}/blob/main/CHANGELOG.md",
|
|
18
|
+
'bug_tracker_uri' => "#{spec.homepage}/issues",
|
|
19
|
+
'documentation_uri' => spec.homepage,
|
|
20
|
+
'rubygems_mfa_required' => 'true'
|
|
21
|
+
}
|
|
22
|
+
|
|
14
23
|
spec.files = `git ls-files -z`.split("\x0")
|
|
15
24
|
spec.executables = %w[shoryuken]
|
|
16
25
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# `shoryuken sqs dump`/`mv` drain a queue via CLI::SQS#find_all. With short
|
|
4
|
+
# polling, real (distributed) SQS routinely returns an empty batch while the
|
|
5
|
+
# queue still has messages, so find_all must use long polling and must not stop
|
|
6
|
+
# on the first empty response - otherwise dump/mv silently process only a
|
|
7
|
+
# fraction of the queue.
|
|
8
|
+
#
|
|
9
|
+
# ElasticMQ never produces those false-empties (nor visibility-timeout re-reads),
|
|
10
|
+
# so this drives find_all with scripted clients to verify the behavior
|
|
11
|
+
# deterministically.
|
|
12
|
+
#
|
|
13
|
+
# Lives under spec/integration (not spec/lib) because requiring bin/cli defines
|
|
14
|
+
# Shoryuken::CLI, and Shoryuken#server? is `defined?(Shoryuken::CLI)` - loading
|
|
15
|
+
# it in the unit suite would flip server? for every other spec. Integration
|
|
16
|
+
# specs each run in their own process, so that's contained here.
|
|
17
|
+
|
|
18
|
+
require 'thor'
|
|
19
|
+
require_relative '../../../bin/cli/base'
|
|
20
|
+
require_relative '../../../bin/cli/sqs'
|
|
21
|
+
|
|
22
|
+
Msg = Struct.new(:message_id, :receipt_handle)
|
|
23
|
+
|
|
24
|
+
# A scripted SQS client: hands out pre-canned batches in order (to simulate
|
|
25
|
+
# false-empties and visibility-timeout re-reads), ignoring max_number_of_messages
|
|
26
|
+
# and recording the wait_time_seconds it was asked for.
|
|
27
|
+
class ScriptedSqsClient
|
|
28
|
+
attr_reader :wait_times
|
|
29
|
+
|
|
30
|
+
def initialize(batches)
|
|
31
|
+
@batches = batches.dup
|
|
32
|
+
@wait_times = []
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def receive_message(params)
|
|
36
|
+
@wait_times << params[:wait_time_seconds]
|
|
37
|
+
Struct.new(:messages).new(@batches.shift || [])
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# A more realistic client backed by a flat list that honours
|
|
42
|
+
# max_number_of_messages, so the finite-limit / batch-sizing path can be driven.
|
|
43
|
+
class FlatSqsClient
|
|
44
|
+
def initialize(messages)
|
|
45
|
+
@messages = messages.dup
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def receive_message(params)
|
|
49
|
+
Struct.new(:messages).new(@messages.shift(params[:max_number_of_messages]))
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def find_all_with(client, limit)
|
|
54
|
+
cli = Shoryuken::CLI::SQS.allocate
|
|
55
|
+
cli.instance_variable_set(:@_sqs, client)
|
|
56
|
+
|
|
57
|
+
collected = []
|
|
58
|
+
count = cli.send(:find_all, 'http://example.com/q', limit) { |msg| collected << msg }
|
|
59
|
+
[collected, count]
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# --- Drains past a false-empty batch, using long polling -------------------
|
|
63
|
+
client = ScriptedSqsClient.new(
|
|
64
|
+
[
|
|
65
|
+
[Msg.new('a'), Msg.new('b'), Msg.new('c')],
|
|
66
|
+
[Msg.new('d'), Msg.new('e')],
|
|
67
|
+
[], # false-empty while 'f' is still queued
|
|
68
|
+
[Msg.new('f')]
|
|
69
|
+
]
|
|
70
|
+
)
|
|
71
|
+
collected, = find_all_with(client, Float::INFINITY)
|
|
72
|
+
|
|
73
|
+
assert_equal(%w[a b c d e f], collected.map(&:message_id), 'find_all should drain past a false-empty batch')
|
|
74
|
+
assert(
|
|
75
|
+
client.wait_times.any? && client.wait_times.all? { |w| w && w.positive? },
|
|
76
|
+
"find_all should use long polling, saw wait_time_seconds: #{client.wait_times.uniq.inspect}"
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
# --- Does not re-yield a message that reappears after its visibility timeout ---
|
|
80
|
+
# 'a' is handed back on a later receive (its visibility timeout lapsed before
|
|
81
|
+
# dump/mv deleted it). find_all must yield and count it only once.
|
|
82
|
+
reappearing = ScriptedSqsClient.new(
|
|
83
|
+
[
|
|
84
|
+
[Msg.new('a'), Msg.new('b')],
|
|
85
|
+
[Msg.new('a')], # re-read of 'a' while it is still in the queue
|
|
86
|
+
[Msg.new('c')]
|
|
87
|
+
]
|
|
88
|
+
)
|
|
89
|
+
collected, count = find_all_with(reappearing, Float::INFINITY)
|
|
90
|
+
|
|
91
|
+
assert_equal(%w[a b c], collected.map(&:message_id), 'find_all must not re-yield a re-read message')
|
|
92
|
+
assert_equal(3, count, 'find_all count must not double-count a re-read message')
|
|
93
|
+
|
|
94
|
+
# --- Retains the newest receipt handle for a re-read message ----------------
|
|
95
|
+
# SQS only honors the most recently received receipt handle for deletion, and
|
|
96
|
+
# dump/mv batch_delete after the drain, so the yielded (and later deleted)
|
|
97
|
+
# message must carry the handle from the latest receive, not the first.
|
|
98
|
+
handles = ScriptedSqsClient.new(
|
|
99
|
+
[
|
|
100
|
+
[Msg.new('x', 'rh-x-1'), Msg.new('y', 'rh-y')],
|
|
101
|
+
[Msg.new('x', 'rh-x-2')], # re-read of 'x' with a fresh handle
|
|
102
|
+
[Msg.new('z', 'rh-z')]
|
|
103
|
+
]
|
|
104
|
+
)
|
|
105
|
+
collected, = find_all_with(handles, Float::INFINITY)
|
|
106
|
+
|
|
107
|
+
x = collected.find { |msg| msg.message_id == 'x' }
|
|
108
|
+
assert_equal('rh-x-2', x.receipt_handle, 'find_all must retain the newest receipt handle for a re-read message')
|
|
109
|
+
assert_equal(%w[x y z], collected.map(&:message_id), 'a re-read must not be yielded twice')
|
|
110
|
+
|
|
111
|
+
# --- Stops at a finite limit without over-fetching -------------------------
|
|
112
|
+
flat = FlatSqsClient.new((1..25).map { |i| Msg.new("m#{i}") })
|
|
113
|
+
collected, count = find_all_with(flat, 15)
|
|
114
|
+
|
|
115
|
+
assert_equal(15, count, 'find_all must stop once the limit is reached')
|
|
116
|
+
assert_equal(
|
|
117
|
+
(1..15).map { |i| "m#{i}" },
|
|
118
|
+
collected.map(&:message_id),
|
|
119
|
+
'find_all must yield exactly the first `limit` messages'
|
|
120
|
+
)
|
|
@@ -17,5 +17,67 @@ RSpec.describe Shoryuken::Client do
|
|
|
17
17
|
expect(Shoryuken::Client.queues(queue_name).url).to eq queue_url
|
|
18
18
|
expect(Shoryuken::Client.queues(queue_name).url).to eq queue_url
|
|
19
19
|
end
|
|
20
|
+
|
|
21
|
+
it 'constructs each queue only once under concurrent first access' do
|
|
22
|
+
allow(described_class).to receive(:sqs).and_return(sqs)
|
|
23
|
+
|
|
24
|
+
construction_count = Shoryuken::Helpers::AtomicCounter.new(0)
|
|
25
|
+
allow(Shoryuken::Queue).to receive(:new) do
|
|
26
|
+
construction_count.increment
|
|
27
|
+
# Mimic the SQS API latency during construction. sleep releases the GVL,
|
|
28
|
+
# so without synchronization every concurrent caller builds its own queue.
|
|
29
|
+
sleep 0.05
|
|
30
|
+
instance_double(Shoryuken::Queue)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
threads = Array.new(10) { Thread.new { described_class.queues('concurrent') } }
|
|
34
|
+
threads.each(&:join)
|
|
35
|
+
|
|
36
|
+
expect(construction_count.value).to eq(1)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
it 'rebuilds cached queues after the sqs client is replaced' do
|
|
40
|
+
allow(described_class).to receive(:sqs).and_return(sqs)
|
|
41
|
+
|
|
42
|
+
construction_count = Shoryuken::Helpers::AtomicCounter.new(0)
|
|
43
|
+
allow(Shoryuken::Queue).to receive(:new) do
|
|
44
|
+
construction_count.increment
|
|
45
|
+
instance_double(Shoryuken::Queue)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
described_class.queues(queue_name)
|
|
49
|
+
described_class.queues(queue_name)
|
|
50
|
+
expect(construction_count.value).to eq(1)
|
|
51
|
+
|
|
52
|
+
described_class.sqs = sqs
|
|
53
|
+
|
|
54
|
+
described_class.queues(queue_name)
|
|
55
|
+
expect(construction_count.value).to eq(2)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
it 'does not cache a queue built with the previous client while it is being replaced' do
|
|
59
|
+
old_sqs = :old_client
|
|
60
|
+
new_sqs = :new_client
|
|
61
|
+
|
|
62
|
+
described_class.sqs = old_sqs
|
|
63
|
+
|
|
64
|
+
allow(Shoryuken::Queue).to receive(:new) do |client, _name|
|
|
65
|
+
instance_double(Shoryuken::Queue, client: client)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Widen the window between resetting the cache and installing the new client so a concurrent
|
|
69
|
+
# `queues` call would cache a queue built with the old client unless the swap is atomic.
|
|
70
|
+
allow(Shoryuken).to receive(:sqs_client=).and_wrap_original do |original, client|
|
|
71
|
+
sleep 0.05
|
|
72
|
+
original.call(client)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
replacer = Thread.new { described_class.sqs = new_sqs }
|
|
76
|
+
sleep 0.01
|
|
77
|
+
reader = Thread.new { described_class.queues('concurrent') }
|
|
78
|
+
[replacer, reader].each(&:join)
|
|
79
|
+
|
|
80
|
+
expect(described_class.queues('concurrent').client).to eq(new_sqs)
|
|
81
|
+
end
|
|
20
82
|
end
|
|
21
83
|
end
|
|
@@ -48,6 +48,17 @@ RSpec.describe Shoryuken::Manager do
|
|
|
48
48
|
expect(waiter.join(5)).to eq(waiter), 'await_dispatching_in_progress deadlocked on a repeated stop'
|
|
49
49
|
end
|
|
50
50
|
|
|
51
|
+
it 'returns when stopped before the dispatch loop ever runs' do
|
|
52
|
+
# No #start here: the dispatch loop never runs, so there is nothing to
|
|
53
|
+
# observe the stop flag and close the signal - stop_new_dispatching must
|
|
54
|
+
# release waiters itself.
|
|
55
|
+
subject.stop_new_dispatching
|
|
56
|
+
|
|
57
|
+
waiter = Thread.new { subject.await_dispatching_in_progress }
|
|
58
|
+
|
|
59
|
+
expect(waiter.join(5)).to eq(waiter), 'await_dispatching_in_progress deadlocked when start never ran'
|
|
60
|
+
end
|
|
61
|
+
|
|
51
62
|
context 'when the executor rejects the dispatch post' do
|
|
52
63
|
let(:executor) do
|
|
53
64
|
double('executor', running?: true).tap do |rejecting_executor|
|
|
@@ -222,6 +233,43 @@ RSpec.describe Shoryuken::Manager do
|
|
|
222
233
|
expect(polling_strategy).to have_received(:message_processed).once
|
|
223
234
|
expect(subject.send(:busy)).to eq(0)
|
|
224
235
|
end
|
|
236
|
+
|
|
237
|
+
context 'when the executor rejects the worker post' do
|
|
238
|
+
# A real ExecutorService instance (Concurrent::Promise validates the
|
|
239
|
+
# executor type), with running? stubbed true so #assign proceeds past its
|
|
240
|
+
# guard and post rejecting - exactly the shutdown race / saturated
|
|
241
|
+
# bounded launcher_executor case. ImmediateExecutor spawns no background
|
|
242
|
+
# thread, so nothing lingers at teardown.
|
|
243
|
+
let(:executor) do
|
|
244
|
+
Concurrent::ImmediateExecutor.new.tap do |rejecting_executor|
|
|
245
|
+
allow(rejecting_executor).to receive(:running?).and_return(true)
|
|
246
|
+
allow(rejecting_executor).to receive(:post).and_raise(Concurrent::RejectedExecutionError)
|
|
247
|
+
end
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
it 'rolls back the busy counter and does not leak it' do
|
|
251
|
+
# The promise body never runs, so completion must not run either
|
|
252
|
+
# (decrement directly, and no FIFO message_processed callback).
|
|
253
|
+
expect(subject).not_to receive(:processor_done)
|
|
254
|
+
|
|
255
|
+
expect { subject.send(:assign, queue, sqs_msg) }.not_to raise_error
|
|
256
|
+
|
|
257
|
+
expect(subject.send(:busy)).to eq(0)
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
it 'does not run the FIFO message_processed callback' do
|
|
261
|
+
# The message was never processed, so the rejection rollback must
|
|
262
|
+
# decrement directly and never reach the FIFO bookkeeping - even for a
|
|
263
|
+
# FIFO queue. Guards against a refactor routing rollback through
|
|
264
|
+
# processor_done.
|
|
265
|
+
allow(sqs_queue).to receive(:fifo?).and_return(true)
|
|
266
|
+
expect(polling_strategy).not_to receive(:message_processed)
|
|
267
|
+
|
|
268
|
+
subject.send(:assign, queue, sqs_msg)
|
|
269
|
+
|
|
270
|
+
expect(subject.send(:busy)).to eq(0)
|
|
271
|
+
end
|
|
272
|
+
end
|
|
225
273
|
end
|
|
226
274
|
|
|
227
275
|
describe '#processor_done' do
|
|
@@ -17,6 +17,21 @@ RSpec.describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
|
|
|
17
17
|
it 'yields' do
|
|
18
18
|
expect { |b| subject.call(TestWorker.new, nil, [], nil, &b) }.to yield_control
|
|
19
19
|
end
|
|
20
|
+
|
|
21
|
+
it 'does not mask the original error when the batch raises' do
|
|
22
|
+
TestWorker.get_shoryuken_options['retry_intervals'] = [300]
|
|
23
|
+
|
|
24
|
+
# Use a logger that actually evaluates lazy log blocks (StringIO, not
|
|
25
|
+
# IO::NULL which skips them), as the default production level does - so a
|
|
26
|
+
# stray message_id/attributes call on the batch Array would surface.
|
|
27
|
+
# Batches aren't backoff-retried, so the original error must propagate
|
|
28
|
+
# rather than a NoMethodError from calling message APIs on the Array.
|
|
29
|
+
allow(subject).to receive(:logger).and_return(Logger.new(StringIO.new, level: Logger::DEBUG))
|
|
30
|
+
|
|
31
|
+
expect {
|
|
32
|
+
subject.call(TestWorker.new, nil, [sqs_msg], nil) { raise 'original batch error' }
|
|
33
|
+
}.to raise_error(RuntimeError, 'original batch error')
|
|
34
|
+
end
|
|
20
35
|
end
|
|
21
36
|
|
|
22
37
|
context 'when no exception' do
|
|
@@ -158,5 +173,22 @@ RSpec.describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
|
|
|
158
173
|
|
|
159
174
|
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
|
|
160
175
|
end
|
|
176
|
+
|
|
177
|
+
it 'never sets a negative visibility timeout for jobs longer than the SQS ceiling' do
|
|
178
|
+
started_at = Time.now - 43_300 # ran longer than the 12h SQS maximum
|
|
179
|
+
|
|
180
|
+
expect(subject.send(:next_visibility_timeout, 300, started_at)).to eq(0)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
it 'does not mask the original error when rescheduling fails' do
|
|
184
|
+
TestWorker.get_shoryuken_options['retry_intervals'] = [300]
|
|
185
|
+
|
|
186
|
+
# e.g. an expired receipt handle - change_visibility raises
|
|
187
|
+
allow(sqs_msg).to receive(:change_visibility).and_raise(StandardError, 'visibility boom')
|
|
188
|
+
|
|
189
|
+
expect {
|
|
190
|
+
subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'original worker error' }
|
|
191
|
+
}.to raise_error(RuntimeError, 'original worker error')
|
|
192
|
+
end
|
|
161
193
|
end
|
|
162
194
|
end
|
|
@@ -187,6 +187,32 @@ RSpec.describe Shoryuken::Queue do
|
|
|
187
187
|
subject.send_message(message_body: 'original')
|
|
188
188
|
end
|
|
189
189
|
end
|
|
190
|
+
|
|
191
|
+
context 'when FIFO' do
|
|
192
|
+
before { allow(subject).to receive(:fifo?).and_return(true) }
|
|
193
|
+
|
|
194
|
+
it 'auto-generates message_group_id and message_deduplication_id by default' do
|
|
195
|
+
expect(sqs).to receive(:send_message) do |arg|
|
|
196
|
+
expect(arg[:message_group_id]).to eq described_class::MESSAGE_GROUP_ID
|
|
197
|
+
expect(arg[:message_deduplication_id]).to be
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
subject.send_message(message_body: 'msg1')
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
context 'and Shoryuken.fifo_message_deduplication is disabled' do
|
|
204
|
+
before { Shoryuken.fifo_message_deduplication = false }
|
|
205
|
+
|
|
206
|
+
it 'does not auto-generate a message_deduplication_id' do
|
|
207
|
+
expect(sqs).to receive(:send_message) do |arg|
|
|
208
|
+
expect(arg[:message_group_id]).to eq described_class::MESSAGE_GROUP_ID
|
|
209
|
+
expect(arg).not_to have_key(:message_deduplication_id)
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
subject.send_message(message_body: 'msg1')
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
end
|
|
190
216
|
end
|
|
191
217
|
|
|
192
218
|
describe '#send_messages' do
|
|
@@ -233,6 +259,26 @@ RSpec.describe Shoryuken::Queue do
|
|
|
233
259
|
|
|
234
260
|
subject.send_messages([{ message_body: 'msg1', message_attributes: { attr: 'attr1' } }])
|
|
235
261
|
end
|
|
262
|
+
|
|
263
|
+
it 'derives the id from the full body, so bodies differing only in an embedded id are not deduplicated' do
|
|
264
|
+
# Guards the ActiveJob opt-out path: when the adapter omits its own id
|
|
265
|
+
# (active_job dedup disabled), this fallback still hashes the full body
|
|
266
|
+
# - which for ActiveJob includes the per-enqueue job_id - so distinct
|
|
267
|
+
# enqueues get distinct ids and are NOT silently collapsed by SQS.
|
|
268
|
+
expect(sqs).to receive(:send_message_batch) do |arg|
|
|
269
|
+
ids = arg[:entries].map { |e| e[:message_deduplication_id] }
|
|
270
|
+
|
|
271
|
+
expect(ids).to all(be)
|
|
272
|
+
expect(ids.uniq.length).to eq(2)
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
subject.send_messages(
|
|
276
|
+
[
|
|
277
|
+
{ message_body: JSON.dump('job_id' => 'a', 'arguments' => ['same']) },
|
|
278
|
+
{ message_body: JSON.dump('job_id' => 'b', 'arguments' => ['same']) }
|
|
279
|
+
]
|
|
280
|
+
)
|
|
281
|
+
end
|
|
236
282
|
end
|
|
237
283
|
|
|
238
284
|
context 'and message_group_id and message_deduplication_id are present' do
|
|
@@ -254,6 +300,33 @@ RSpec.describe Shoryuken::Queue do
|
|
|
254
300
|
)
|
|
255
301
|
end
|
|
256
302
|
end
|
|
303
|
+
|
|
304
|
+
context 'and Shoryuken.fifo_message_deduplication is disabled' do
|
|
305
|
+
before { Shoryuken.fifo_message_deduplication = false }
|
|
306
|
+
|
|
307
|
+
it 'does not auto-generate a message_deduplication_id' do
|
|
308
|
+
expect(sqs).to receive(:send_message_batch) do |arg|
|
|
309
|
+
first_entry = arg[:entries].first
|
|
310
|
+
|
|
311
|
+
expect(first_entry[:message_group_id]).to eq described_class::MESSAGE_GROUP_ID
|
|
312
|
+
expect(first_entry).not_to have_key(:message_deduplication_id)
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
subject.send_messages([{ message_body: 'msg1', message_attributes: { attr: 'attr1' } }])
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
it 'still honors an explicit message_deduplication_id' do
|
|
319
|
+
expect(sqs).to receive(:send_message_batch) do |arg|
|
|
320
|
+
first_entry = arg[:entries].first
|
|
321
|
+
|
|
322
|
+
expect(first_entry[:message_deduplication_id]).to eq 'my id'
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
subject.send_messages(
|
|
326
|
+
[{ message_body: 'msg1', message_attributes: { attr: 'attr1' }, message_deduplication_id: 'my id' }]
|
|
327
|
+
)
|
|
328
|
+
end
|
|
329
|
+
end
|
|
257
330
|
end
|
|
258
331
|
|
|
259
332
|
it 'accepts an array of string' do
|
|
@@ -265,6 +338,19 @@ RSpec.describe Shoryuken::Queue do
|
|
|
265
338
|
|
|
266
339
|
subject.send_messages(%w[msg1 msg2])
|
|
267
340
|
end
|
|
341
|
+
|
|
342
|
+
context 'when some sends fail' do
|
|
343
|
+
it 'logs each failure (so bulk/enqueue_all drops are not silent)' do
|
|
344
|
+
failure = double(id: '1', code: 'code', message: '...', sender_fault: false)
|
|
345
|
+
logger = double('Logger')
|
|
346
|
+
allow(sqs).to receive(:send_message_batch).and_return(double(failed: [failure]))
|
|
347
|
+
allow(subject).to receive(:logger).and_return(logger)
|
|
348
|
+
|
|
349
|
+
expect(logger).to receive(:error)
|
|
350
|
+
|
|
351
|
+
subject.send_messages(entries: [{ id: '0', message_body: 'msg1' }])
|
|
352
|
+
end
|
|
353
|
+
end
|
|
268
354
|
end
|
|
269
355
|
|
|
270
356
|
describe '#fifo?' do
|
|
@@ -101,5 +101,18 @@ RSpec.describe Shoryuken::Worker::DefaultExecutor do
|
|
|
101
101
|
|
|
102
102
|
TestWorker.perform_async('delayed message', queue: new_queue)
|
|
103
103
|
end
|
|
104
|
+
|
|
105
|
+
it 'does not mutate the caller-supplied options hash' do
|
|
106
|
+
allow(sqs_queue).to receive(:send_message)
|
|
107
|
+
|
|
108
|
+
options = { queue: queue, message_attributes: { 'custom' => { string_value: 'v', data_type: 'String' } } }
|
|
109
|
+
|
|
110
|
+
TestWorker.perform_async('message', options)
|
|
111
|
+
|
|
112
|
+
# :queue must not be deleted, :message_body must not be injected, and the
|
|
113
|
+
# nested message_attributes must not gain shoryuken_class - otherwise a
|
|
114
|
+
# reused options hash would reroute/corrupt later enqueues.
|
|
115
|
+
expect(options).to eq(queue: queue, message_attributes: { 'custom' => { string_value: 'v', data_type: 'String' } })
|
|
116
|
+
end
|
|
104
117
|
end
|
|
105
118
|
end
|
|
@@ -25,6 +25,18 @@ RSpec.describe Shoryuken::Worker::InlineExecutor do
|
|
|
25
25
|
|
|
26
26
|
TestWorker.perform_async('test', message_attributes: custom_attributes)
|
|
27
27
|
end
|
|
28
|
+
|
|
29
|
+
it 'does not mutate the caller-supplied options hash' do
|
|
30
|
+
allow_any_instance_of(TestWorker).to receive(:perform)
|
|
31
|
+
|
|
32
|
+
options = { queue: 'a_queue', message_attributes: { 'custom' => { string_value: 'v', data_type: 'String' } } }
|
|
33
|
+
|
|
34
|
+
TestWorker.perform_async('test', options)
|
|
35
|
+
|
|
36
|
+
# :queue and :message_attributes must survive intact so a reused options
|
|
37
|
+
# hash routes later jobs correctly instead of falling back to the default.
|
|
38
|
+
expect(options).to eq(queue: 'a_queue', message_attributes: { 'custom' => { string_value: 'v', data_type: 'String' } })
|
|
39
|
+
end
|
|
28
40
|
end
|
|
29
41
|
|
|
30
42
|
describe '.perform_in' do
|
data/spec/spec_helper.rb
CHANGED
|
@@ -43,23 +43,23 @@ Dotenv.load
|
|
|
43
43
|
unless ENV['SIMPLECOV_DISABLED']
|
|
44
44
|
require 'simplecov'
|
|
45
45
|
SimpleCov.start do
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
46
|
+
skip '/spec/'
|
|
47
|
+
skip '/test_workers/'
|
|
48
|
+
skip '/examples/'
|
|
49
|
+
skip '/vendor/'
|
|
50
|
+
skip '/.bundle/'
|
|
51
|
+
|
|
52
|
+
group 'Library', 'lib/'
|
|
53
|
+
group 'ActiveJob', 'lib/active_job'
|
|
54
|
+
group 'Middleware', 'lib/shoryuken/middleware'
|
|
55
|
+
group 'Polling', 'lib/shoryuken/polling'
|
|
56
|
+
group 'Workers', 'lib/shoryuken/worker'
|
|
57
|
+
group 'Helpers', 'lib/shoryuken/helpers'
|
|
58
58
|
|
|
59
59
|
enable_coverage :branch
|
|
60
60
|
|
|
61
61
|
minimum_coverage 89
|
|
62
|
-
|
|
62
|
+
coverage(:line) { minimum 60, per: :file }
|
|
63
63
|
end
|
|
64
64
|
end
|
|
65
65
|
|
|
@@ -117,6 +117,7 @@ RSpec.configure do |config|
|
|
|
117
117
|
|
|
118
118
|
Shoryuken.active_job_queue_name_prefixing = false
|
|
119
119
|
Shoryuken.active_job_fifo_message_deduplication = true
|
|
120
|
+
Shoryuken.fifo_message_deduplication = true
|
|
120
121
|
|
|
121
122
|
Shoryuken.worker_registry.clear
|
|
122
123
|
Shoryuken.register_worker('default', TestWorker)
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: shoryuken
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 7.0.
|
|
4
|
+
version: 7.0.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Pablo Cantero
|
|
@@ -247,6 +247,7 @@ files:
|
|
|
247
247
|
- spec/integration/body_parser/json_parser_spec.rb
|
|
248
248
|
- spec/integration/body_parser/proc_parser_spec.rb
|
|
249
249
|
- spec/integration/body_parser/text_parser_spec.rb
|
|
250
|
+
- spec/integration/cli/find_all_spec.rb
|
|
250
251
|
- spec/integration/concurrent_processing/concurrent_processing_spec.rb
|
|
251
252
|
- spec/integration/concurrent_processing/processor_accounting_spec.rb
|
|
252
253
|
- spec/integration/custom_group_polling_strategy/custom_group_polling_strategy_spec.rb
|
|
@@ -341,7 +342,13 @@ files:
|
|
|
341
342
|
homepage: https://github.com/ruby-shoryuken/shoryuken
|
|
342
343
|
licenses:
|
|
343
344
|
- LGPL-3.0
|
|
344
|
-
metadata:
|
|
345
|
+
metadata:
|
|
346
|
+
homepage_uri: https://github.com/ruby-shoryuken/shoryuken
|
|
347
|
+
source_code_uri: https://github.com/ruby-shoryuken/shoryuken
|
|
348
|
+
changelog_uri: https://github.com/ruby-shoryuken/shoryuken/blob/main/CHANGELOG.md
|
|
349
|
+
bug_tracker_uri: https://github.com/ruby-shoryuken/shoryuken/issues
|
|
350
|
+
documentation_uri: https://github.com/ruby-shoryuken/shoryuken
|
|
351
|
+
rubygems_mfa_required: 'true'
|
|
345
352
|
rdoc_options: []
|
|
346
353
|
require_paths:
|
|
347
354
|
- lib
|
|
@@ -356,7 +363,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
356
363
|
- !ruby/object:Gem::Version
|
|
357
364
|
version: '0'
|
|
358
365
|
requirements: []
|
|
359
|
-
rubygems_version: 4.0.
|
|
366
|
+
rubygems_version: 4.0.20
|
|
360
367
|
specification_version: 4
|
|
361
368
|
summary: Shoryuken is a super efficient AWS SQS thread based message processor
|
|
362
369
|
test_files:
|
|
@@ -389,6 +396,7 @@ test_files:
|
|
|
389
396
|
- spec/integration/body_parser/json_parser_spec.rb
|
|
390
397
|
- spec/integration/body_parser/proc_parser_spec.rb
|
|
391
398
|
- spec/integration/body_parser/text_parser_spec.rb
|
|
399
|
+
- spec/integration/cli/find_all_spec.rb
|
|
392
400
|
- spec/integration/concurrent_processing/concurrent_processing_spec.rb
|
|
393
401
|
- spec/integration/concurrent_processing/processor_accounting_spec.rb
|
|
394
402
|
- spec/integration/custom_group_polling_strategy/custom_group_polling_strategy_spec.rb
|