shoryuken 7.0.3 → 7.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/push.yml +3 -3
  3. data/.github/workflows/specs.yml +4 -4
  4. data/.github/workflows/verify-action-pins.yml +1 -1
  5. data/.ruby-version +1 -1
  6. data/.yard-lint.yml +29 -1
  7. data/CHANGELOG.md +65 -1
  8. data/Gemfile.lint.lock +8 -7
  9. data/bin/cli/sqs.rb +60 -3
  10. data/lib/active_job/extensions.rb +5 -8
  11. data/lib/active_job/queue_adapters/shoryuken_adapter.rb +3 -4
  12. data/lib/shoryuken/active_job/current_attributes.rb +4 -6
  13. data/lib/shoryuken/body_parser.rb +1 -2
  14. data/lib/shoryuken/client.rb +15 -6
  15. data/lib/shoryuken/default_exception_handler.rb +2 -3
  16. data/lib/shoryuken/errors.rb +2 -4
  17. data/lib/shoryuken/helpers/atomic_counter.rb +6 -9
  18. data/lib/shoryuken/helpers/atomic_hash.rb +9 -15
  19. data/lib/shoryuken/helpers/hash_utils.rb +5 -7
  20. data/lib/shoryuken/helpers/string_utils.rb +6 -8
  21. data/lib/shoryuken/inline_message.rb +3 -4
  22. data/lib/shoryuken/launcher.rb +8 -13
  23. data/lib/shoryuken/manager.rb +27 -1
  24. data/lib/shoryuken/message.rb +2 -3
  25. data/lib/shoryuken/middleware/chain.rb +7 -13
  26. data/lib/shoryuken/middleware/entry.rb +1 -2
  27. data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +3 -5
  28. data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +38 -21
  29. data/lib/shoryuken/middleware/server/timing.rb +2 -3
  30. data/lib/shoryuken/options.rb +28 -6
  31. data/lib/shoryuken/polling/base_strategy.rb +16 -23
  32. data/lib/shoryuken/polling/queue_configuration.rb +8 -12
  33. data/lib/shoryuken/polling/weighted_round_robin.rb +3 -4
  34. data/lib/shoryuken/queue.rb +33 -3
  35. data/lib/shoryuken/version.rb +1 -1
  36. data/lib/shoryuken/worker/default_executor.rb +11 -5
  37. data/lib/shoryuken/worker/inline_executor.rb +10 -5
  38. data/lib/shoryuken/worker.rb +10 -12
  39. data/lib/shoryuken.rb +4 -3
  40. data/shoryuken.gemspec +9 -0
  41. data/spec/integration/cli/find_all_spec.rb +120 -0
  42. data/spec/lib/shoryuken/client_spec.rb +62 -0
  43. data/spec/lib/shoryuken/manager_spec.rb +48 -0
  44. data/spec/lib/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +32 -0
  45. data/spec/lib/shoryuken/queue_spec.rb +86 -0
  46. data/spec/lib/shoryuken/worker/default_executor_spec.rb +13 -0
  47. data/spec/lib/shoryuken/worker/inline_executor_spec.rb +12 -0
  48. data/spec/spec_helper.rb +14 -13
  49. metadata +11 -3
@@ -4,10 +4,9 @@ module Shoryuken
4
4
  # Worker module provides the core functionality for creating Shoryuken workers
5
5
  # that process messages from Amazon SQS queues.
6
6
  #
7
- # Including this module in a class provides methods for configuring queue processing,
8
- # enqueueing jobs, and setting up middleware. Workers can be configured for different
9
- # processing patterns including single message processing, batch processing, and
10
- # various retry and visibility timeout strategies.
7
+ # Including this module in a class provides methods for configuring queue processing, enqueueing jobs, and setting up
8
+ # middleware. Workers can be configured for different processing patterns including single message processing, batch
9
+ # processing, and various retry and visibility timeout strategies.
11
10
  #
12
11
  # @example Basic worker implementation
13
12
  # class EmailWorker
@@ -123,7 +122,8 @@ module Shoryuken
123
122
  # @option opts [Boolean] :auto_delete (false) Automatically delete messages after processing
124
123
  # @option opts [Boolean] :auto_visibility_timeout (false) Automatically extend message visibility
125
124
  # @option opts [Array<Integer>] :retry_intervals Exponential backoff retry intervals in seconds
126
- # @option opts [Array<Class>, Proc] :non_retryable_exceptions Exception classes or lambda that should skip retries and delete message immediately
125
+ # @option opts [Array<Class>, Proc] :non_retryable_exceptions Exception classes or lambda that should
126
+ # skip retries and delete message immediately
127
127
  # @option opts [Hash] :sqs Additional SQS client options
128
128
  #
129
129
  # @example Basic worker configuration
@@ -216,9 +216,8 @@ module Shoryuken
216
216
  !!get_shoryuken_options['auto_visibility_timeout']
217
217
  end
218
218
 
219
- # Checks if exponential backoff retry is configured for this worker.
220
- # When retry intervals are specified, failed jobs will be retried with
221
- # increasing delays between attempts.
219
+ # Checks if exponential backoff retry is configured for this worker. When retry intervals are specified, failed jobs
220
+ # will be retried with increasing delays between attempts.
222
221
  #
223
222
  # @return [Boolean] true if retry intervals are configured
224
223
  #
@@ -235,10 +234,9 @@ module Shoryuken
235
234
  !!get_shoryuken_options['retry_intervals']
236
235
  end
237
236
 
238
- # Checks if automatic message deletion is enabled for this worker.
239
- # When enabled, successfully processed messages are automatically deleted
240
- # from the SQS queue. When disabled, you must manually delete messages
241
- # or they will become visible again after the visibility timeout.
237
+ # Checks if automatic message deletion is enabled for this worker. When enabled, successfully processed messages are
238
+ # automatically deleted from the SQS queue. When disabled, you must manually delete messages or they will become
239
+ # visible again after the visibility timeout.
242
240
  #
243
241
  # @return [Boolean] true if auto delete is enabled
244
242
  #
data/lib/shoryuken.rb CHANGED
@@ -27,9 +27,8 @@ module Shoryuken
27
27
  @_shoryuken_options ||= Shoryuken::Options.new
28
28
  end
29
29
 
30
- # Checks if the Shoryuken server is running and healthy.
31
- # A server is considered healthy when all configured processing groups
32
- # are running and able to process messages.
30
+ # Checks if the Shoryuken server is running and healthy. A server is considered healthy when all configured processing
31
+ # groups are running and able to process messages.
33
32
  #
34
33
  # @return [Boolean] true if the server is healthy
35
34
  def self.healthy?
@@ -60,6 +59,8 @@ module Shoryuken
60
59
  :active_job_queue_name_prefixing=,
61
60
  :active_job_fifo_message_deduplication?,
62
61
  :active_job_fifo_message_deduplication=,
62
+ :fifo_message_deduplication?,
63
+ :fifo_message_deduplication=,
63
64
  :sqs_client,
64
65
  :sqs_client=,
65
66
  :sqs_client_receive_message_opts,
data/shoryuken.gemspec CHANGED
@@ -11,6 +11,15 @@ Gem::Specification.new do |spec|
11
11
  spec.homepage = 'https://github.com/ruby-shoryuken/shoryuken'
12
12
  spec.license = 'LGPL-3.0'
13
13
 
14
+ spec.metadata = {
15
+ 'homepage_uri' => spec.homepage,
16
+ 'source_code_uri' => spec.homepage,
17
+ 'changelog_uri' => "#{spec.homepage}/blob/main/CHANGELOG.md",
18
+ 'bug_tracker_uri' => "#{spec.homepage}/issues",
19
+ 'documentation_uri' => spec.homepage,
20
+ 'rubygems_mfa_required' => 'true'
21
+ }
22
+
14
23
  spec.files = `git ls-files -z`.split("\x0")
15
24
  spec.executables = %w[shoryuken]
16
25
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
@@ -0,0 +1,120 @@
1
+ # frozen_string_literal: true
2
+
3
+ # `shoryuken sqs dump`/`mv` drain a queue via CLI::SQS#find_all. With short
4
+ # polling, real (distributed) SQS routinely returns an empty batch while the
5
+ # queue still has messages, so find_all must use long polling and must not stop
6
+ # on the first empty response - otherwise dump/mv silently process only a
7
+ # fraction of the queue.
8
+ #
9
+ # ElasticMQ never produces those false-empties (nor visibility-timeout re-reads),
10
+ # so this drives find_all with scripted clients to verify the behavior
11
+ # deterministically.
12
+ #
13
+ # Lives under spec/integration (not spec/lib) because requiring bin/cli defines
14
+ # Shoryuken::CLI, and Shoryuken#server? is `defined?(Shoryuken::CLI)` - loading
15
+ # it in the unit suite would flip server? for every other spec. Integration
16
+ # specs each run in their own process, so that's contained here.
17
+
18
+ require 'thor'
19
+ require_relative '../../../bin/cli/base'
20
+ require_relative '../../../bin/cli/sqs'
21
+
22
+ Msg = Struct.new(:message_id, :receipt_handle)
23
+
24
+ # A scripted SQS client: hands out pre-canned batches in order (to simulate
25
+ # false-empties and visibility-timeout re-reads), ignoring max_number_of_messages
26
+ # and recording the wait_time_seconds it was asked for.
27
+ class ScriptedSqsClient
28
+ attr_reader :wait_times
29
+
30
+ def initialize(batches)
31
+ @batches = batches.dup
32
+ @wait_times = []
33
+ end
34
+
35
+ def receive_message(params)
36
+ @wait_times << params[:wait_time_seconds]
37
+ Struct.new(:messages).new(@batches.shift || [])
38
+ end
39
+ end
40
+
41
+ # A more realistic client backed by a flat list that honours
42
+ # max_number_of_messages, so the finite-limit / batch-sizing path can be driven.
43
+ class FlatSqsClient
44
+ def initialize(messages)
45
+ @messages = messages.dup
46
+ end
47
+
48
+ def receive_message(params)
49
+ Struct.new(:messages).new(@messages.shift(params[:max_number_of_messages]))
50
+ end
51
+ end
52
+
53
+ def find_all_with(client, limit)
54
+ cli = Shoryuken::CLI::SQS.allocate
55
+ cli.instance_variable_set(:@_sqs, client)
56
+
57
+ collected = []
58
+ count = cli.send(:find_all, 'http://example.com/q', limit) { |msg| collected << msg }
59
+ [collected, count]
60
+ end
61
+
62
+ # --- Drains past a false-empty batch, using long polling -------------------
63
+ client = ScriptedSqsClient.new(
64
+ [
65
+ [Msg.new('a'), Msg.new('b'), Msg.new('c')],
66
+ [Msg.new('d'), Msg.new('e')],
67
+ [], # false-empty while 'f' is still queued
68
+ [Msg.new('f')]
69
+ ]
70
+ )
71
+ collected, = find_all_with(client, Float::INFINITY)
72
+
73
+ assert_equal(%w[a b c d e f], collected.map(&:message_id), 'find_all should drain past a false-empty batch')
74
+ assert(
75
+ client.wait_times.any? && client.wait_times.all? { |w| w && w.positive? },
76
+ "find_all should use long polling, saw wait_time_seconds: #{client.wait_times.uniq.inspect}"
77
+ )
78
+
79
+ # --- Does not re-yield a message that reappears after its visibility timeout ---
80
+ # 'a' is handed back on a later receive (its visibility timeout lapsed before
81
+ # dump/mv deleted it). find_all must yield and count it only once.
82
+ reappearing = ScriptedSqsClient.new(
83
+ [
84
+ [Msg.new('a'), Msg.new('b')],
85
+ [Msg.new('a')], # re-read of 'a' while it is still in the queue
86
+ [Msg.new('c')]
87
+ ]
88
+ )
89
+ collected, count = find_all_with(reappearing, Float::INFINITY)
90
+
91
+ assert_equal(%w[a b c], collected.map(&:message_id), 'find_all must not re-yield a re-read message')
92
+ assert_equal(3, count, 'find_all count must not double-count a re-read message')
93
+
94
+ # --- Retains the newest receipt handle for a re-read message ----------------
95
+ # SQS only honors the most recently received receipt handle for deletion, and
96
+ # dump/mv batch_delete after the drain, so the yielded (and later deleted)
97
+ # message must carry the handle from the latest receive, not the first.
98
+ handles = ScriptedSqsClient.new(
99
+ [
100
+ [Msg.new('x', 'rh-x-1'), Msg.new('y', 'rh-y')],
101
+ [Msg.new('x', 'rh-x-2')], # re-read of 'x' with a fresh handle
102
+ [Msg.new('z', 'rh-z')]
103
+ ]
104
+ )
105
+ collected, = find_all_with(handles, Float::INFINITY)
106
+
107
+ x = collected.find { |msg| msg.message_id == 'x' }
108
+ assert_equal('rh-x-2', x.receipt_handle, 'find_all must retain the newest receipt handle for a re-read message')
109
+ assert_equal(%w[x y z], collected.map(&:message_id), 'a re-read must not be yielded twice')
110
+
111
+ # --- Stops at a finite limit without over-fetching -------------------------
112
+ flat = FlatSqsClient.new((1..25).map { |i| Msg.new("m#{i}") })
113
+ collected, count = find_all_with(flat, 15)
114
+
115
+ assert_equal(15, count, 'find_all must stop once the limit is reached')
116
+ assert_equal(
117
+ (1..15).map { |i| "m#{i}" },
118
+ collected.map(&:message_id),
119
+ 'find_all must yield exactly the first `limit` messages'
120
+ )
@@ -17,5 +17,67 @@ RSpec.describe Shoryuken::Client do
17
17
  expect(Shoryuken::Client.queues(queue_name).url).to eq queue_url
18
18
  expect(Shoryuken::Client.queues(queue_name).url).to eq queue_url
19
19
  end
20
+
21
+ it 'constructs each queue only once under concurrent first access' do
22
+ allow(described_class).to receive(:sqs).and_return(sqs)
23
+
24
+ construction_count = Shoryuken::Helpers::AtomicCounter.new(0)
25
+ allow(Shoryuken::Queue).to receive(:new) do
26
+ construction_count.increment
27
+ # Mimic the SQS API latency during construction. sleep releases the GVL,
28
+ # so without synchronization every concurrent caller builds its own queue.
29
+ sleep 0.05
30
+ instance_double(Shoryuken::Queue)
31
+ end
32
+
33
+ threads = Array.new(10) { Thread.new { described_class.queues('concurrent') } }
34
+ threads.each(&:join)
35
+
36
+ expect(construction_count.value).to eq(1)
37
+ end
38
+
39
+ it 'rebuilds cached queues after the sqs client is replaced' do
40
+ allow(described_class).to receive(:sqs).and_return(sqs)
41
+
42
+ construction_count = Shoryuken::Helpers::AtomicCounter.new(0)
43
+ allow(Shoryuken::Queue).to receive(:new) do
44
+ construction_count.increment
45
+ instance_double(Shoryuken::Queue)
46
+ end
47
+
48
+ described_class.queues(queue_name)
49
+ described_class.queues(queue_name)
50
+ expect(construction_count.value).to eq(1)
51
+
52
+ described_class.sqs = sqs
53
+
54
+ described_class.queues(queue_name)
55
+ expect(construction_count.value).to eq(2)
56
+ end
57
+
58
+ it 'does not cache a queue built with the previous client while it is being replaced' do
59
+ old_sqs = :old_client
60
+ new_sqs = :new_client
61
+
62
+ described_class.sqs = old_sqs
63
+
64
+ allow(Shoryuken::Queue).to receive(:new) do |client, _name|
65
+ instance_double(Shoryuken::Queue, client: client)
66
+ end
67
+
68
+ # Widen the window between resetting the cache and installing the new client so a concurrent
69
+ # `queues` call would cache a queue built with the old client unless the swap is atomic.
70
+ allow(Shoryuken).to receive(:sqs_client=).and_wrap_original do |original, client|
71
+ sleep 0.05
72
+ original.call(client)
73
+ end
74
+
75
+ replacer = Thread.new { described_class.sqs = new_sqs }
76
+ sleep 0.01
77
+ reader = Thread.new { described_class.queues('concurrent') }
78
+ [replacer, reader].each(&:join)
79
+
80
+ expect(described_class.queues('concurrent').client).to eq(new_sqs)
81
+ end
20
82
  end
21
83
  end
@@ -48,6 +48,17 @@ RSpec.describe Shoryuken::Manager do
48
48
  expect(waiter.join(5)).to eq(waiter), 'await_dispatching_in_progress deadlocked on a repeated stop'
49
49
  end
50
50
 
51
+ it 'returns when stopped before the dispatch loop ever runs' do
52
+ # No #start here: the dispatch loop never runs, so there is nothing to
53
+ # observe the stop flag and close the signal - stop_new_dispatching must
54
+ # release waiters itself.
55
+ subject.stop_new_dispatching
56
+
57
+ waiter = Thread.new { subject.await_dispatching_in_progress }
58
+
59
+ expect(waiter.join(5)).to eq(waiter), 'await_dispatching_in_progress deadlocked when start never ran'
60
+ end
61
+
51
62
  context 'when the executor rejects the dispatch post' do
52
63
  let(:executor) do
53
64
  double('executor', running?: true).tap do |rejecting_executor|
@@ -222,6 +233,43 @@ RSpec.describe Shoryuken::Manager do
222
233
  expect(polling_strategy).to have_received(:message_processed).once
223
234
  expect(subject.send(:busy)).to eq(0)
224
235
  end
236
+
237
+ context 'when the executor rejects the worker post' do
238
+ # A real ExecutorService instance (Concurrent::Promise validates the
239
+ # executor type), with running? stubbed true so #assign proceeds past its
240
+ # guard and post rejecting - exactly the shutdown race / saturated
241
+ # bounded launcher_executor case. ImmediateExecutor spawns no background
242
+ # thread, so nothing lingers at teardown.
243
+ let(:executor) do
244
+ Concurrent::ImmediateExecutor.new.tap do |rejecting_executor|
245
+ allow(rejecting_executor).to receive(:running?).and_return(true)
246
+ allow(rejecting_executor).to receive(:post).and_raise(Concurrent::RejectedExecutionError)
247
+ end
248
+ end
249
+
250
+ it 'rolls back the busy counter and does not leak it' do
251
+ # The promise body never runs, so completion must not run either
252
+ # (decrement directly, and no FIFO message_processed callback).
253
+ expect(subject).not_to receive(:processor_done)
254
+
255
+ expect { subject.send(:assign, queue, sqs_msg) }.not_to raise_error
256
+
257
+ expect(subject.send(:busy)).to eq(0)
258
+ end
259
+
260
+ it 'does not run the FIFO message_processed callback' do
261
+ # The message was never processed, so the rejection rollback must
262
+ # decrement directly and never reach the FIFO bookkeeping - even for a
263
+ # FIFO queue. Guards against a refactor routing rollback through
264
+ # processor_done.
265
+ allow(sqs_queue).to receive(:fifo?).and_return(true)
266
+ expect(polling_strategy).not_to receive(:message_processed)
267
+
268
+ subject.send(:assign, queue, sqs_msg)
269
+
270
+ expect(subject.send(:busy)).to eq(0)
271
+ end
272
+ end
225
273
  end
226
274
 
227
275
  describe '#processor_done' do
@@ -17,6 +17,21 @@ RSpec.describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
17
17
  it 'yields' do
18
18
  expect { |b| subject.call(TestWorker.new, nil, [], nil, &b) }.to yield_control
19
19
  end
20
+
21
+ it 'does not mask the original error when the batch raises' do
22
+ TestWorker.get_shoryuken_options['retry_intervals'] = [300]
23
+
24
+ # Use a logger that actually evaluates lazy log blocks (StringIO, not
25
+ # IO::NULL which skips them), as the default production level does - so a
26
+ # stray message_id/attributes call on the batch Array would surface.
27
+ # Batches aren't backoff-retried, so the original error must propagate
28
+ # rather than a NoMethodError from calling message APIs on the Array.
29
+ allow(subject).to receive(:logger).and_return(Logger.new(StringIO.new, level: Logger::DEBUG))
30
+
31
+ expect {
32
+ subject.call(TestWorker.new, nil, [sqs_msg], nil) { raise 'original batch error' }
33
+ }.to raise_error(RuntimeError, 'original batch error')
34
+ end
20
35
  end
21
36
 
22
37
  context 'when no exception' do
@@ -158,5 +173,22 @@ RSpec.describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
158
173
 
159
174
  expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
160
175
  end
176
+
177
+ it 'never sets a negative visibility timeout for jobs longer than the SQS ceiling' do
178
+ started_at = Time.now - 43_300 # ran longer than the 12h SQS maximum
179
+
180
+ expect(subject.send(:next_visibility_timeout, 300, started_at)).to eq(0)
181
+ end
182
+
183
+ it 'does not mask the original error when rescheduling fails' do
184
+ TestWorker.get_shoryuken_options['retry_intervals'] = [300]
185
+
186
+ # e.g. an expired receipt handle - change_visibility raises
187
+ allow(sqs_msg).to receive(:change_visibility).and_raise(StandardError, 'visibility boom')
188
+
189
+ expect {
190
+ subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'original worker error' }
191
+ }.to raise_error(RuntimeError, 'original worker error')
192
+ end
161
193
  end
162
194
  end
@@ -187,6 +187,32 @@ RSpec.describe Shoryuken::Queue do
187
187
  subject.send_message(message_body: 'original')
188
188
  end
189
189
  end
190
+
191
+ context 'when FIFO' do
192
+ before { allow(subject).to receive(:fifo?).and_return(true) }
193
+
194
+ it 'auto-generates message_group_id and message_deduplication_id by default' do
195
+ expect(sqs).to receive(:send_message) do |arg|
196
+ expect(arg[:message_group_id]).to eq described_class::MESSAGE_GROUP_ID
197
+ expect(arg[:message_deduplication_id]).to be
198
+ end
199
+
200
+ subject.send_message(message_body: 'msg1')
201
+ end
202
+
203
+ context 'and Shoryuken.fifo_message_deduplication is disabled' do
204
+ before { Shoryuken.fifo_message_deduplication = false }
205
+
206
+ it 'does not auto-generate a message_deduplication_id' do
207
+ expect(sqs).to receive(:send_message) do |arg|
208
+ expect(arg[:message_group_id]).to eq described_class::MESSAGE_GROUP_ID
209
+ expect(arg).not_to have_key(:message_deduplication_id)
210
+ end
211
+
212
+ subject.send_message(message_body: 'msg1')
213
+ end
214
+ end
215
+ end
190
216
  end
191
217
 
192
218
  describe '#send_messages' do
@@ -233,6 +259,26 @@ RSpec.describe Shoryuken::Queue do
233
259
 
234
260
  subject.send_messages([{ message_body: 'msg1', message_attributes: { attr: 'attr1' } }])
235
261
  end
262
+
263
+ it 'derives the id from the full body, so bodies differing only in an embedded id are not deduplicated' do
264
+ # Guards the ActiveJob opt-out path: when the adapter omits its own id
265
+ # (active_job dedup disabled), this fallback still hashes the full body
266
+ # - which for ActiveJob includes the per-enqueue job_id - so distinct
267
+ # enqueues get distinct ids and are NOT silently collapsed by SQS.
268
+ expect(sqs).to receive(:send_message_batch) do |arg|
269
+ ids = arg[:entries].map { |e| e[:message_deduplication_id] }
270
+
271
+ expect(ids).to all(be)
272
+ expect(ids.uniq.length).to eq(2)
273
+ end
274
+
275
+ subject.send_messages(
276
+ [
277
+ { message_body: JSON.dump('job_id' => 'a', 'arguments' => ['same']) },
278
+ { message_body: JSON.dump('job_id' => 'b', 'arguments' => ['same']) }
279
+ ]
280
+ )
281
+ end
236
282
  end
237
283
 
238
284
  context 'and message_group_id and message_deduplication_id are present' do
@@ -254,6 +300,33 @@ RSpec.describe Shoryuken::Queue do
254
300
  )
255
301
  end
256
302
  end
303
+
304
+ context 'and Shoryuken.fifo_message_deduplication is disabled' do
305
+ before { Shoryuken.fifo_message_deduplication = false }
306
+
307
+ it 'does not auto-generate a message_deduplication_id' do
308
+ expect(sqs).to receive(:send_message_batch) do |arg|
309
+ first_entry = arg[:entries].first
310
+
311
+ expect(first_entry[:message_group_id]).to eq described_class::MESSAGE_GROUP_ID
312
+ expect(first_entry).not_to have_key(:message_deduplication_id)
313
+ end
314
+
315
+ subject.send_messages([{ message_body: 'msg1', message_attributes: { attr: 'attr1' } }])
316
+ end
317
+
318
+ it 'still honors an explicit message_deduplication_id' do
319
+ expect(sqs).to receive(:send_message_batch) do |arg|
320
+ first_entry = arg[:entries].first
321
+
322
+ expect(first_entry[:message_deduplication_id]).to eq 'my id'
323
+ end
324
+
325
+ subject.send_messages(
326
+ [{ message_body: 'msg1', message_attributes: { attr: 'attr1' }, message_deduplication_id: 'my id' }]
327
+ )
328
+ end
329
+ end
257
330
  end
258
331
 
259
332
  it 'accepts an array of string' do
@@ -265,6 +338,19 @@ RSpec.describe Shoryuken::Queue do
265
338
 
266
339
  subject.send_messages(%w[msg1 msg2])
267
340
  end
341
+
342
+ context 'when some sends fail' do
343
+ it 'logs each failure (so bulk/enqueue_all drops are not silent)' do
344
+ failure = double(id: '1', code: 'code', message: '...', sender_fault: false)
345
+ logger = double('Logger')
346
+ allow(sqs).to receive(:send_message_batch).and_return(double(failed: [failure]))
347
+ allow(subject).to receive(:logger).and_return(logger)
348
+
349
+ expect(logger).to receive(:error)
350
+
351
+ subject.send_messages(entries: [{ id: '0', message_body: 'msg1' }])
352
+ end
353
+ end
268
354
  end
269
355
 
270
356
  describe '#fifo?' do
@@ -101,5 +101,18 @@ RSpec.describe Shoryuken::Worker::DefaultExecutor do
101
101
 
102
102
  TestWorker.perform_async('delayed message', queue: new_queue)
103
103
  end
104
+
105
+ it 'does not mutate the caller-supplied options hash' do
106
+ allow(sqs_queue).to receive(:send_message)
107
+
108
+ options = { queue: queue, message_attributes: { 'custom' => { string_value: 'v', data_type: 'String' } } }
109
+
110
+ TestWorker.perform_async('message', options)
111
+
112
+ # :queue must not be deleted, :message_body must not be injected, and the
113
+ # nested message_attributes must not gain shoryuken_class - otherwise a
114
+ # reused options hash would reroute/corrupt later enqueues.
115
+ expect(options).to eq(queue: queue, message_attributes: { 'custom' => { string_value: 'v', data_type: 'String' } })
116
+ end
104
117
  end
105
118
  end
@@ -25,6 +25,18 @@ RSpec.describe Shoryuken::Worker::InlineExecutor do
25
25
 
26
26
  TestWorker.perform_async('test', message_attributes: custom_attributes)
27
27
  end
28
+
29
+ it 'does not mutate the caller-supplied options hash' do
30
+ allow_any_instance_of(TestWorker).to receive(:perform)
31
+
32
+ options = { queue: 'a_queue', message_attributes: { 'custom' => { string_value: 'v', data_type: 'String' } } }
33
+
34
+ TestWorker.perform_async('test', options)
35
+
36
+ # :queue and :message_attributes must survive intact so a reused options
37
+ # hash routes later jobs correctly instead of falling back to the default.
38
+ expect(options).to eq(queue: 'a_queue', message_attributes: { 'custom' => { string_value: 'v', data_type: 'String' } })
39
+ end
28
40
  end
29
41
 
30
42
  describe '.perform_in' do
data/spec/spec_helper.rb CHANGED
@@ -43,23 +43,23 @@ Dotenv.load
43
43
  unless ENV['SIMPLECOV_DISABLED']
44
44
  require 'simplecov'
45
45
  SimpleCov.start do
46
- add_filter '/spec/'
47
- add_filter '/test_workers/'
48
- add_filter '/examples/'
49
- add_filter '/vendor/'
50
- add_filter '/.bundle/'
51
-
52
- add_group 'Library', 'lib/'
53
- add_group 'ActiveJob', 'lib/active_job'
54
- add_group 'Middleware', 'lib/shoryuken/middleware'
55
- add_group 'Polling', 'lib/shoryuken/polling'
56
- add_group 'Workers', 'lib/shoryuken/worker'
57
- add_group 'Helpers', 'lib/shoryuken/helpers'
46
+ skip '/spec/'
47
+ skip '/test_workers/'
48
+ skip '/examples/'
49
+ skip '/vendor/'
50
+ skip '/.bundle/'
51
+
52
+ group 'Library', 'lib/'
53
+ group 'ActiveJob', 'lib/active_job'
54
+ group 'Middleware', 'lib/shoryuken/middleware'
55
+ group 'Polling', 'lib/shoryuken/polling'
56
+ group 'Workers', 'lib/shoryuken/worker'
57
+ group 'Helpers', 'lib/shoryuken/helpers'
58
58
 
59
59
  enable_coverage :branch
60
60
 
61
61
  minimum_coverage 89
62
- minimum_coverage_by_file 60
62
+ coverage(:line) { minimum 60, per: :file }
63
63
  end
64
64
  end
65
65
 
@@ -117,6 +117,7 @@ RSpec.configure do |config|
117
117
 
118
118
  Shoryuken.active_job_queue_name_prefixing = false
119
119
  Shoryuken.active_job_fifo_message_deduplication = true
120
+ Shoryuken.fifo_message_deduplication = true
120
121
 
121
122
  Shoryuken.worker_registry.clear
122
123
  Shoryuken.register_worker('default', TestWorker)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: shoryuken
3
3
  version: !ruby/object:Gem::Version
4
- version: 7.0.3
4
+ version: 7.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Pablo Cantero
@@ -247,6 +247,7 @@ files:
247
247
  - spec/integration/body_parser/json_parser_spec.rb
248
248
  - spec/integration/body_parser/proc_parser_spec.rb
249
249
  - spec/integration/body_parser/text_parser_spec.rb
250
+ - spec/integration/cli/find_all_spec.rb
250
251
  - spec/integration/concurrent_processing/concurrent_processing_spec.rb
251
252
  - spec/integration/concurrent_processing/processor_accounting_spec.rb
252
253
  - spec/integration/custom_group_polling_strategy/custom_group_polling_strategy_spec.rb
@@ -341,7 +342,13 @@ files:
341
342
  homepage: https://github.com/ruby-shoryuken/shoryuken
342
343
  licenses:
343
344
  - LGPL-3.0
344
- metadata: {}
345
+ metadata:
346
+ homepage_uri: https://github.com/ruby-shoryuken/shoryuken
347
+ source_code_uri: https://github.com/ruby-shoryuken/shoryuken
348
+ changelog_uri: https://github.com/ruby-shoryuken/shoryuken/blob/main/CHANGELOG.md
349
+ bug_tracker_uri: https://github.com/ruby-shoryuken/shoryuken/issues
350
+ documentation_uri: https://github.com/ruby-shoryuken/shoryuken
351
+ rubygems_mfa_required: 'true'
345
352
  rdoc_options: []
346
353
  require_paths:
347
354
  - lib
@@ -356,7 +363,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
356
363
  - !ruby/object:Gem::Version
357
364
  version: '0'
358
365
  requirements: []
359
- rubygems_version: 4.0.10
366
+ rubygems_version: 4.0.20
360
367
  specification_version: 4
361
368
  summary: Shoryuken is a super efficient AWS SQS thread based message processor
362
369
  test_files:
@@ -389,6 +396,7 @@ test_files:
389
396
  - spec/integration/body_parser/json_parser_spec.rb
390
397
  - spec/integration/body_parser/proc_parser_spec.rb
391
398
  - spec/integration/body_parser/text_parser_spec.rb
399
+ - spec/integration/cli/find_all_spec.rb
392
400
  - spec/integration/concurrent_processing/concurrent_processing_spec.rb
393
401
  - spec/integration/concurrent_processing/processor_accounting_spec.rb
394
402
  - spec/integration/custom_group_polling_strategy/custom_group_polling_strategy_spec.rb