shoryuken 7.0.3 → 7.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/push.yml +3 -3
  3. data/.github/workflows/specs.yml +4 -4
  4. data/.github/workflows/verify-action-pins.yml +1 -1
  5. data/.ruby-version +1 -1
  6. data/.yard-lint.yml +29 -1
  7. data/CHANGELOG.md +65 -1
  8. data/Gemfile.lint.lock +8 -7
  9. data/bin/cli/sqs.rb +60 -3
  10. data/lib/active_job/extensions.rb +5 -8
  11. data/lib/active_job/queue_adapters/shoryuken_adapter.rb +3 -4
  12. data/lib/shoryuken/active_job/current_attributes.rb +4 -6
  13. data/lib/shoryuken/body_parser.rb +1 -2
  14. data/lib/shoryuken/client.rb +15 -6
  15. data/lib/shoryuken/default_exception_handler.rb +2 -3
  16. data/lib/shoryuken/errors.rb +2 -4
  17. data/lib/shoryuken/helpers/atomic_counter.rb +6 -9
  18. data/lib/shoryuken/helpers/atomic_hash.rb +9 -15
  19. data/lib/shoryuken/helpers/hash_utils.rb +5 -7
  20. data/lib/shoryuken/helpers/string_utils.rb +6 -8
  21. data/lib/shoryuken/inline_message.rb +3 -4
  22. data/lib/shoryuken/launcher.rb +8 -13
  23. data/lib/shoryuken/manager.rb +27 -1
  24. data/lib/shoryuken/message.rb +2 -3
  25. data/lib/shoryuken/middleware/chain.rb +7 -13
  26. data/lib/shoryuken/middleware/entry.rb +1 -2
  27. data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +3 -5
  28. data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +38 -21
  29. data/lib/shoryuken/middleware/server/timing.rb +2 -3
  30. data/lib/shoryuken/options.rb +28 -6
  31. data/lib/shoryuken/polling/base_strategy.rb +16 -23
  32. data/lib/shoryuken/polling/queue_configuration.rb +8 -12
  33. data/lib/shoryuken/polling/weighted_round_robin.rb +3 -4
  34. data/lib/shoryuken/queue.rb +33 -3
  35. data/lib/shoryuken/version.rb +1 -1
  36. data/lib/shoryuken/worker/default_executor.rb +11 -5
  37. data/lib/shoryuken/worker/inline_executor.rb +10 -5
  38. data/lib/shoryuken/worker.rb +10 -12
  39. data/lib/shoryuken.rb +4 -3
  40. data/shoryuken.gemspec +9 -0
  41. data/spec/integration/cli/find_all_spec.rb +120 -0
  42. data/spec/lib/shoryuken/client_spec.rb +62 -0
  43. data/spec/lib/shoryuken/manager_spec.rb +48 -0
  44. data/spec/lib/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +32 -0
  45. data/spec/lib/shoryuken/queue_spec.rb +86 -0
  46. data/spec/lib/shoryuken/worker/default_executor_spec.rb +13 -0
  47. data/spec/lib/shoryuken/worker/inline_executor_spec.rb +12 -0
  48. data/spec/spec_helper.rb +14 -13
  49. metadata +11 -3
@@ -14,9 +14,8 @@ module Shoryuken
14
14
 
15
15
  # Indicates whether the launcher is in the process of stopping.
16
16
  #
17
- # This flag is set to true when either {#stop} or {#stop!} is called,
18
- # and is used by ActiveJob adapters to signal jobs that they should
19
- # checkpoint and prepare for graceful shutdown.
17
+ # This flag is set to true when either {#stop} or {#stop!} is called, and is used by ActiveJob adapters to signal jobs
18
+ # that they should checkpoint and prepare for graceful shutdown.
20
19
  #
21
20
  # @return [Boolean] true if stopping, false otherwise
22
21
  def stopping?
@@ -91,11 +90,9 @@ module Shoryuken
91
90
  @managers.each(&:await_dispatching_in_progress)
92
91
  end
93
92
 
94
- # Shuts the executor down, giving in-flight workers up to the configured
95
- # timeout to finish before force-killing them so the process can exit.
96
- # Used by both the graceful ({#stop}) and immediate ({#stop!}) shutdowns:
97
- # a graceful stop still waits for workers, but must not block forever on a
98
- # hung one.
93
+ # Shuts the executor down, giving in-flight workers up to the configured timeout to finish before force-killing them so
94
+ # the process can exit. Used by both the graceful ({#stop}) and immediate ({#stop!}) shutdowns: a graceful stop still
95
+ # waits for workers, but must not block forever on a hung one.
99
96
  #
100
97
  # @return [void]
101
98
  def shutdown_executor
@@ -105,11 +102,9 @@ module Shoryuken
105
102
 
106
103
  # Returns the executor for running async operations
107
104
  #
108
- # Owns a dedicated executor rather than borrowing Concurrent.global_io_executor:
109
- # {#stop} and {#stop!} shut down and kill this executor, and destroying the
110
- # process-global pool would break anything else relying on it (including
111
- # Shoryuken's own ShoryukenConcurrentSendAdapter) and prevent a fresh launcher
112
- # from starting in the same process.
105
+ # Owns a dedicated executor rather than borrowing Concurrent.global_io_executor: {#stop} and {#stop!} shut down and kill
106
+ # this executor, and destroying the process-global pool would break anything else relying on it (including Shoryuken's
107
+ # own ShoryukenConcurrentSendAdapter) and prevent a fresh launcher from starting in the same process.
113
108
  #
114
109
  # @return [Concurrent::ExecutorService] the executor service
115
110
  def executor
@@ -32,6 +32,7 @@ module Shoryuken
32
32
  @executor = executor
33
33
  @running = Shoryuken::Helpers::AtomicBoolean.new(true)
34
34
  @stop_new_dispatching = Shoryuken::Helpers::AtomicBoolean.new(false)
35
+ @dispatch_started = Shoryuken::Helpers::AtomicBoolean.new(false)
35
36
  @dispatching_release_signal = ::Queue.new
36
37
  end
37
38
 
@@ -48,6 +49,14 @@ module Shoryuken
48
49
  # @return [void]
49
50
  def stop_new_dispatching
50
51
  @stop_new_dispatching.make_true
52
+
53
+ # If the dispatch loop never ran (e.g. a stop arrives before the start
54
+ # Future is scheduled, or in an embedded host whose executor is
55
+ # saturated), there is no loop to observe the flag and close the release
56
+ # signal, so await_dispatching_in_progress would block forever. Close it
57
+ # here in that case. Queue#close is idempotent, so a dispatch_loop that
58
+ # does start later and closes it again is harmless.
59
+ @dispatching_release_signal.close unless @dispatch_started.true?
51
60
  end
52
61
 
53
62
  # Waits for any in-progress dispatching to complete
@@ -76,6 +85,11 @@ module Shoryuken
76
85
  #
77
86
  # @return [void]
78
87
  def dispatch_loop
88
+ # Mark that the loop has run at least once so stop_new_dispatching knows
89
+ # it can rely on the loop to close the release signal (and otherwise
90
+ # closes it itself to avoid a deadlock).
91
+ @dispatch_started.make_true
92
+
79
93
  if @stop_new_dispatching.true? || !running?
80
94
  # Close (instead of push) so every pending and future
81
95
  # await_dispatching_in_progress call returns, not just the first one
@@ -156,7 +170,8 @@ module Shoryuken
156
170
  #
157
171
  # @param queue_name [String] the queue name
158
172
  # @param sqs_msg [Aws::SQS::Types::Message, Array<Aws::SQS::Types::Message>] the message or batch
159
- # @return [Concurrent::Promise, nil] the processing promise or nil if not running
173
+ # @return [Concurrent::Promise, nil] the processing promise, or nil if the manager
174
+ # is not running or the executor rejects the worker post
160
175
  def assign(queue_name, sqs_msg)
161
176
  return unless running?
162
177
 
@@ -181,6 +196,17 @@ module Shoryuken
181
196
  processor_done(queue_name)
182
197
  end
183
198
  end
199
+ rescue Concurrent::RejectedExecutionError
200
+ # The executor was shut down (or a bounded custom launcher_executor is
201
+ # saturated) between the running? check above and the post. The promise
202
+ # body - and therefore processor_done - never ran, so roll back the
203
+ # increment here. Leaking it would permanently shrink `ready`
204
+ # (@max_processors - busy) until dispatch stalls and the group stops
205
+ # processing. The message was never processed, so we must not run the
206
+ # FIFO message_processed callback - decrement directly instead.
207
+ @busy_processors.decrement
208
+ fire_utilization_update_event
209
+ nil
184
210
  end
185
211
 
186
212
  # Dispatches a batch of messages from a queue
@@ -1,9 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Shoryuken
4
- # Represents an SQS message received by a Shoryuken worker.
5
- # This class wraps the raw AWS SQS message data and provides convenient methods
6
- # for interacting with the message, including deletion and visibility timeout management.
4
+ # Represents an SQS message received by a Shoryuken worker. This class wraps the raw AWS SQS message data and provides
5
+ # convenient methods for interacting with the message, including deletion and visibility timeout management.
7
6
  #
8
7
  # Message instances are automatically created by Shoryuken and passed to your
9
8
  # worker's `perform` method as the first argument.
@@ -1,10 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Shoryuken
4
- # Middleware provides a way to wrap message processing with custom logic,
5
- # similar to Rack middleware in web applications. Middleware runs on the server
6
- # side and can perform setup, teardown, error handling, and monitoring around
7
- # job execution.
4
+ # Middleware provides a way to wrap message processing with custom logic, similar to Rack middleware in web applications.
5
+ # Middleware runs on the server side and can perform setup, teardown, error handling, and monitoring around job execution.
8
6
  #
9
7
  # Middleware classes must implement a `call` method that accepts the worker instance,
10
8
  # queue name, and SQS message, and must yield to continue the middleware chain.
@@ -84,9 +82,8 @@ module Shoryuken
84
82
  # @see Shoryuken::Middleware::Chain Middleware chain management
85
83
  # @see https://github.com/ruby-shoryuken/shoryuken/wiki/Middleware Comprehensive middleware guide
86
84
  module Middleware
87
- # Manages a chain of middleware classes that will be instantiated and invoked
88
- # in sequence around message processing. Provides methods for adding, removing,
89
- # and reordering middleware.
85
+ # Manages a chain of middleware classes that will be instantiated and invoked in sequence around message processing.
86
+ # Provides methods for adding, removing, and reordering middleware.
90
87
  class Chain
91
88
  # @return [Array<Entry>] The ordered list of middleware entries
92
89
  attr_reader :entries
@@ -121,8 +118,7 @@ module Shoryuken
121
118
  entries.delete_if { |entry| entry.klass == klass }
122
119
  end
123
120
 
124
- # Adds middleware to the end of the chain. Does nothing if the middleware
125
- # class is already present in the chain.
121
+ # Adds middleware to the end of the chain. Does nothing if the middleware class is already present in the chain.
126
122
  #
127
123
  # @param klass [Class] The middleware class to add
128
124
  # @param args [Array] Arguments to pass to the middleware constructor
@@ -132,8 +128,7 @@ module Shoryuken
132
128
  entries << Entry.new(klass, *args) unless exists?(klass)
133
129
  end
134
130
 
135
- # Adds middleware to the beginning of the chain. Does nothing if the middleware
136
- # class is already present in the chain.
131
+ # Adds middleware to the beginning of the chain. Does nothing if the middleware class is already present in the chain.
137
132
  #
138
133
  # @param klass [Class] The middleware class to prepend
139
134
  # @param args [Array] Arguments to pass to the middleware constructor
@@ -195,8 +190,7 @@ module Shoryuken
195
190
  entries.clear
196
191
  end
197
192
 
198
- # Invokes the middleware chain with the given arguments.
199
- # Each middleware's call method will be invoked in sequence,
193
+ # Invokes the middleware chain with the given arguments. Each middleware's call method will be invoked in sequence,
200
194
  # with control passed through yielding.
201
195
  #
202
196
  # @param args [Array] arguments to pass to each middleware
@@ -2,8 +2,7 @@
2
2
 
3
3
  module Shoryuken
4
4
  module Middleware
5
- # Represents an entry in a middleware chain, storing the middleware class
6
- # and any arguments needed for its instantiation.
5
+ # Represents an entry in a middleware chain, storing the middleware class and any arguments needed for its instantiation.
7
6
  #
8
7
  # @api private
9
8
  class Entry
@@ -73,11 +73,9 @@ module Shoryuken
73
73
 
74
74
  private
75
75
 
76
- # Returns a positive interval at which to re-extend the message's
77
- # visibility before it expires, or nil when the visibility timeout is
78
- # too short to schedule one. Normally this is EXTEND_UPFRONT_SECONDS
79
- # before expiry, but for short timeouts (<= EXTEND_UPFRONT_SECONDS) it
80
- # falls back to half the timeout so the timer still fires in time
76
+ # Returns a positive interval at which to re-extend the message's visibility before it expires, or nil when the
77
+ # visibility timeout is too short to schedule one. Normally this is EXTEND_UPFRONT_SECONDS before expiry, but for
78
+ # short timeouts (<= EXTEND_UPFRONT_SECONDS) it falls back to half the timeout so the timer still fires in time
81
79
  # instead of TimerTask raising on a non-positive interval.
82
80
  #
83
81
  # @param visibility_timeout [Integer] the queue's visibility timeout in seconds
@@ -3,9 +3,8 @@
3
3
  module Shoryuken
4
4
  module Middleware
5
5
  module Server
6
- # Middleware that implements exponential backoff retry for failed messages.
7
- # When a job fails, the message visibility timeout is adjusted based on
8
- # configured retry intervals.
6
+ # Middleware that implements exponential backoff retry for failed messages. When a job fails, the message visibility
7
+ # timeout is adjusted based on configured retry intervals.
9
8
  class ExponentialBackoffRetry
10
9
  include Util
11
10
 
@@ -28,24 +27,31 @@ module Shoryuken
28
27
  end
29
28
 
30
29
  started_at = Time.now
31
- yield
32
- rescue => e
33
- worker_options = worker.class.get_shoryuken_options
34
- retry_intervals = worker_options['retry_intervals']
35
-
36
- # Non-retryable exceptions must not be backoff retried; re-raise so the
37
- # NonRetryableException middleware can delete the message immediately.
38
- raise if NonRetryableException.non_retryable?(e, worker_options['non_retryable_exceptions'])
39
30
 
40
- if retry_intervals.nil? || !handle_failure(sqs_msg, started_at, retry_intervals)
41
- # Re-raise the exception if the job is not going to be exponential backoff retried.
42
- # This allows custom middleware (like exception notifiers) to be aware of the unhandled failure.
43
- raise
31
+ # Scope the rescue to the single-message yield only. A method-level
32
+ # rescue would also catch the batch `return yield` above, then route the
33
+ # Array into handle_failure (which calls #attributes/#message_id on it),
34
+ # raising a NoMethodError that masks the original worker error.
35
+ begin
36
+ yield
37
+ rescue => e
38
+ worker_options = worker.class.get_shoryuken_options
39
+ retry_intervals = worker_options['retry_intervals']
40
+
41
+ # Non-retryable exceptions must not be backoff retried; re-raise so the
42
+ # NonRetryableException middleware can delete the message immediately.
43
+ raise if NonRetryableException.non_retryable?(e, worker_options['non_retryable_exceptions'])
44
+
45
+ if retry_intervals.nil? || !handle_failure(sqs_msg, started_at, retry_intervals)
46
+ # Re-raise the exception if the job is not going to be exponential backoff retried.
47
+ # This allows custom middleware (like exception notifiers) to be aware of the unhandled failure.
48
+ raise
49
+ end
50
+
51
+ logger.warn { "Message #{sqs_msg.message_id} will attempt retry due to error: #{e.message}" }
52
+ # since we didn't raise, lets log the backtrace for debugging purposes.
53
+ logger.debug { e.backtrace.join("\n") } unless e.backtrace.nil?
44
54
  end
45
-
46
- logger.warn { "Message #{sqs_msg.message_id} will attempt retry due to error: #{e.message}" }
47
- # since we didn't raise, lets log the backtrace for debugging purposes.
48
- logger.debug { e.backtrace.join("\n") } unless e.backtrace.nil?
49
55
  end
50
56
 
51
57
  private
@@ -69,11 +75,14 @@ module Shoryuken
69
75
  #
70
76
  # @param interval [Integer] the desired interval
71
77
  # @param started_at [Time] when processing started
72
- # @return [Integer] the capped visibility timeout
78
+ # @return [Integer] the capped visibility timeout (never negative)
73
79
  def next_visibility_timeout(interval, started_at)
74
80
  max_timeout = 43_200 - (Time.now - started_at).ceil - 1
75
81
  interval = max_timeout if interval > max_timeout
76
- interval.to_i
82
+ # Never go negative: a job that ran longer than the 12h SQS ceiling
83
+ # drives max_timeout below zero, and change_message_visibility rejects
84
+ # a negative timeout. Clamp to 0 (retry as soon as possible) instead.
85
+ [interval.to_i, 0].max
77
86
  end
78
87
 
79
88
  # Handles a message failure by adjusting visibility timeout
@@ -92,6 +101,14 @@ module Shoryuken
92
101
  logger.info { "Message #{sqs_msg.message_id} failed, will be retried in #{interval} seconds" }
93
102
 
94
103
  true
104
+ rescue => e
105
+ # Rescheduling itself failed (e.g. an expired receipt handle, or a
106
+ # transient SQS error). Don't let that exception escape and mask the
107
+ # original worker failure - log it and report "not retried" so the
108
+ # caller re-raises the original error and the message falls back to the
109
+ # queue's default visibility timeout.
110
+ logger.warn { "Failed to set backoff visibility timeout for #{sqs_msg.message_id}: #{e.message}" }
111
+ false
95
112
  end
96
113
  end
97
114
  end
@@ -3,9 +3,8 @@
3
3
  module Shoryuken
4
4
  module Middleware
5
5
  module Server
6
- # Middleware that logs timing information for message processing.
7
- # Records start time, completion time, and warns if processing
8
- # exceeds the queue's visibility timeout.
6
+ # Middleware that logs timing information for message processing. Records start time, completion time, and warns if
7
+ # processing exceeds the queue's visibility timeout.
9
8
  class Timing
10
9
  include Util
11
10
 
@@ -1,9 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Shoryuken
4
- # Stores and manages all Shoryuken configuration options.
5
- # This class is used internally to hold settings for workers, queues,
6
- # middleware, and other runtime configurations.
4
+ # Stores and manages all Shoryuken configuration options. This class is used internally to hold settings for workers,
5
+ # queues, middleware, and other runtime configurations.
7
6
  class Options
8
7
  # Default configuration values for Shoryuken
9
8
  DEFAULTS = {
@@ -35,6 +34,7 @@ module Shoryuken
35
34
  # @return [Shoryuken::WorkerRegistry] the registry for worker classes
36
35
  # @return [Array<#call>] handlers for processing exceptions
37
36
  attr_accessor :active_job_queue_name_prefixing, :active_job_fifo_message_deduplication,
37
+ :fifo_message_deduplication,
38
38
  :cache_visibility_timeout,
39
39
  :groups, :launcher_executor, :reloader, :enable_reloading,
40
40
  :start_callback, :stop_callback, :worker_executor, :worker_registry,
@@ -55,6 +55,7 @@ module Shoryuken
55
55
  self.exception_handlers = [DefaultExceptionHandler]
56
56
  self.active_job_queue_name_prefixing = false
57
57
  self.active_job_fifo_message_deduplication = true
58
+ self.fifo_message_deduplication = true
58
59
  self.worker_executor = Worker::DefaultExecutor
59
60
  self.cache_visibility_timeout = false
60
61
  self.reloader = proc { |&block| block.call }
@@ -304,15 +305,36 @@ module Shoryuken
304
305
  @active_job_queue_name_prefixing
305
306
  end
306
307
 
307
- # Checks if the ActiveJob adapter should auto-generate a content-based
308
- # message_deduplication_id for FIFO queues. When disabled, distinct enqueues
309
- # of the same job class and arguments are no longer silently deduplicated.
308
+ # Checks if the ActiveJob adapter should auto-generate a content-based message_deduplication_id for FIFO queues. When
309
+ # disabled, distinct enqueues of the same job class and arguments are no longer silently deduplicated.
310
310
  #
311
311
  # @return [Boolean] true if FIFO deduplication id generation is enabled
312
312
  def active_job_fifo_message_deduplication?
313
313
  @active_job_fifo_message_deduplication
314
314
  end
315
315
 
316
+ # Checks whether Shoryuken auto-generates a content-based
317
+ # message_deduplication_id for raw FIFO sends (Worker.perform_async,
318
+ # Queue#send_message / #send_messages). When disabled, distinct sends of an
319
+ # identical body are no longer silently deduplicated by SQS - you must then
320
+ # provide a message_deduplication_id yourself or enable
321
+ # ContentBasedDeduplication on the queue.
322
+ #
323
+ # ActiveJob enqueues are primarily controlled by
324
+ # #active_job_fifo_message_deduplication?. When that is enabled the adapter
325
+ # sets a content-based message_deduplication_id itself (from the body minus
326
+ # job_id/enqueued_at) before the raw send, so this flag is a no-op for them.
327
+ # When it is disabled the adapter omits the id and this flag governs the
328
+ # fallback: left enabled (the default) each enqueue still gets a hash of the
329
+ # full serialized body - which includes the per-enqueue job_id, so distinct
330
+ # enqueues are NOT deduplicated - while disabling it drops the id entirely
331
+ # (then requiring an explicit id or the queue's ContentBasedDeduplication).
332
+ #
333
+ # @return [Boolean] true if FIFO deduplication id generation is enabled
334
+ def fifo_message_deduplication?
335
+ @fifo_message_deduplication
336
+ end
337
+
316
338
  private
317
339
 
318
340
  # Creates the default server middleware chain
@@ -5,10 +5,9 @@ module Shoryuken
5
5
  module Polling
6
6
  # Abstract base class for queue polling strategies.
7
7
  #
8
- # This class defines the interface that all polling strategies must implement
9
- # to manage queue selection and message flow control in Shoryuken workers.
10
- # Polling strategies determine which queue to fetch messages from next and
11
- # how to handle scenarios where queues have no messages available.
8
+ # This class defines the interface that all polling strategies must implement to manage queue selection and message flow
9
+ # control in Shoryuken workers. Polling strategies determine which queue to fetch messages from next and how to handle
10
+ # scenarios where queues have no messages available.
12
11
  #
13
12
  # @abstract Subclass and override {#next_queue}, {#messages_found}, and {#active_queues}
14
13
  # to implement a custom polling strategy.
@@ -42,9 +41,8 @@ module Shoryuken
42
41
 
43
42
  # Returns the next queue to poll for messages.
44
43
  #
45
- # This method should return a QueueConfiguration object representing
46
- # the next queue that should be polled for messages, or nil if no
47
- # queues are currently available for polling.
44
+ # This method should return a QueueConfiguration object representing the next queue that should be polled for messages,
45
+ # or nil if no queues are currently available for polling.
48
46
  #
49
47
  # @abstract Subclasses must implement this method
50
48
  # @return [QueueConfiguration, nil] Next queue to poll, or nil if none available
@@ -55,10 +53,9 @@ module Shoryuken
55
53
 
56
54
  # Called when messages are found (or not found) in a queue.
57
55
  #
58
- # This method is invoked after polling a queue to inform the strategy
59
- # about the number of messages that were retrieved. Strategies can use
60
- # this information to make decisions about future polling behavior,
61
- # such as pausing empty queues or adjusting queue weights.
56
+ # This method is invoked after polling a queue to inform the strategy about the number of messages that were retrieved.
57
+ # Strategies can use this information to make decisions about future polling behavior, such as pausing empty queues or
58
+ # adjusting queue weights.
62
59
  #
63
60
  # @abstract Subclasses must implement this method
64
61
  # @param _queue [String] The name of the queue that was polled
@@ -70,9 +67,8 @@ module Shoryuken
70
67
 
71
68
  # Called when a message from a queue has been processed.
72
69
  #
73
- # This optional callback is invoked after a message has been successfully
74
- # processed by a worker. Strategies can use this information for cleanup
75
- # or to adjust their polling behavior.
70
+ # This optional callback is invoked after a message has been successfully processed by a worker. Strategies can use
71
+ # this information for cleanup or to adjust their polling behavior.
76
72
  #
77
73
  # @param _queue [String] The name of the queue whose message was processed
78
74
  # @return [void]
@@ -80,9 +76,8 @@ module Shoryuken
80
76
 
81
77
  # Returns the list of currently active queues.
82
78
  #
83
- # This method should return an array representing the queues that are
84
- # currently active and available for polling. The format may vary by
85
- # strategy implementation.
79
+ # This method should return an array representing the queues that are currently active and available for polling. The
80
+ # format may vary by strategy implementation.
86
81
  #
87
82
  # @abstract Subclasses must implement this method
88
83
  # @return [Array] List of active queues
@@ -93,9 +88,8 @@ module Shoryuken
93
88
 
94
89
  # Compares this strategy with another object for equality.
95
90
  #
96
- # Two strategies are considered equal if they have the same active queues.
97
- # This method also supports comparison with Array objects for backward
98
- # compatibility.
91
+ # Two strategies are considered equal if they have the same active queues. This method also supports comparison with
92
+ # Array objects for backward compatibility.
99
93
  #
100
94
  # @param other [Object] Object to compare with
101
95
  # @return [Boolean] true if strategies are equivalent
@@ -114,9 +108,8 @@ module Shoryuken
114
108
 
115
109
  # Returns the delay time for pausing empty queues.
116
110
  #
117
- # This method returns the amount of time (in seconds) that empty queues
118
- # should be paused before being polled again. The delay can be set at
119
- # the strategy level or falls back to the global Shoryuken delay setting.
111
+ # This method returns the amount of time (in seconds) that empty queues should be paused before being polled again. The
112
+ # delay can be set at the strategy level or falls back to the global Shoryuken delay setting.
120
113
  #
121
114
  # @return [Float] Delay time in seconds
122
115
  def delay
@@ -4,9 +4,8 @@ module Shoryuken
4
4
  module Polling
5
5
  # Configuration object representing a queue and its associated options.
6
6
  #
7
- # This class encapsulates a queue name along with any polling-specific
8
- # options or metadata. It provides a structured way to pass queue
9
- # information between polling strategies and the message fetching system.
7
+ # This class encapsulates a queue name along with any polling-specific options or metadata. It provides a structured way
8
+ # to pass queue information between polling strategies and the message fetching system.
10
9
  #
11
10
  # The class extends Struct to provide attribute accessors for name and options
12
11
  # while adding custom behavior for equality comparison and string representation.
@@ -32,9 +31,8 @@ module Shoryuken
32
31
  QueueConfiguration = Struct.new(:name, :options) do
33
32
  # Generates a hash value based on the queue name.
34
33
  #
35
- # This method ensures that QueueConfiguration objects can be used
36
- # as hash keys and that configurations with the same queue name
37
- # will have the same hash value regardless of their options.
34
+ # This method ensures that QueueConfiguration objects can be used as hash keys and that configurations with the same
35
+ # queue name will have the same hash value regardless of their options.
38
36
  #
39
37
  # @return [Integer] Hash value based on the queue name
40
38
  def hash
@@ -43,9 +41,8 @@ module Shoryuken
43
41
 
44
42
  # Compares this configuration with another object for equality.
45
43
  #
46
- # Two QueueConfiguration objects are equal if they have the same name
47
- # and options. For convenience, a configuration with empty options can
48
- # also be compared directly with a string queue name.
44
+ # Two QueueConfiguration objects are equal if they have the same name and options. For convenience, a configuration
45
+ # with empty options can also be compared directly with a string queue name.
49
46
  #
50
47
  # @param other [Object] The object to compare with
51
48
  # @return [Boolean] true if the objects are considered equal
@@ -78,9 +75,8 @@ module Shoryuken
78
75
 
79
76
  # Returns a string representation of the queue configuration.
80
77
  #
81
- # For configurations with empty options, returns just the queue name.
82
- # For configurations with options, returns a detailed representation
83
- # showing both the name and the options hash.
78
+ # For configurations with empty options, returns just the queue name. For configurations with options, returns a
79
+ # detailed representation showing both the name and the options hash.
84
80
  #
85
81
  # @return [String] String representation of the configuration
86
82
  #
@@ -90,10 +90,9 @@ module Shoryuken
90
90
 
91
91
  # Unpauses the first queue whose delay has expired.
92
92
  #
93
- # Scans for any expired entry rather than only the head of the list:
94
- # message_processed marks a queue ready by setting its time to the epoch,
95
- # and that entry may sit behind an earlier-paused queue that is still
96
- # paused - checking only the head would leave the ready queue stuck.
93
+ # Scans for any expired entry rather than only the head of the list: message_processed marks a queue ready by setting
94
+ # its time to the epoch, and that entry may sit behind an earlier-paused queue that is still paused - checking only the
95
+ # head would leave the ready queue stuck.
97
96
  #
98
97
  # @return [void]
99
98
  def unpause_queues
@@ -89,7 +89,11 @@ module Shoryuken
89
89
  # @option options [Array<Hash>] :entries message entries to send
90
90
  # @return [Aws::SQS::Types::SendMessageBatchResult] the batch send result
91
91
  def send_messages(options)
92
- client.send_message_batch(sanitize_messages!(options).merge(queue_url: url))
92
+ response = client.send_message_batch(sanitize_messages!(options).merge(queue_url: url))
93
+
94
+ log_failed_sends(response)
95
+
96
+ response
93
97
  end
94
98
 
95
99
  # Receives messages from the queue
@@ -120,6 +124,24 @@ module Shoryuken
120
124
 
121
125
  private
122
126
 
127
+ # Logs any per-entry failures from a batch send. SQS reports these in
128
+ # response.failed (throttling, oversize/malformed entries, or a same-batch
129
+ # duplicate message_deduplication_id) rather than raising, so without this
130
+ # they are silently swallowed - notably by ShoryukenAdapter#enqueue_all,
131
+ # which reads only response.successful. Mirrors delete_messages.
132
+ #
133
+ # @param response [Aws::SQS::Types::SendMessageBatchResult] the batch result
134
+ # @return [void]
135
+ def log_failed_sends(response)
136
+ return unless response.respond_to?(:failed)
137
+
138
+ Array(response.failed).each do |failure|
139
+ logger.error do
140
+ "Could not send #{failure.id}, code: '#{failure.code}', message: '#{failure.message}', sender_fault: #{failure.sender_fault}"
141
+ end
142
+ end
143
+ end
144
+
123
145
  # Initializes the FIFO attribute by calling fifo?
124
146
  #
125
147
  # @return [Boolean] whether the queue is FIFO
@@ -224,8 +246,16 @@ module Shoryuken
224
246
  def add_fifo_attributes!(options)
225
247
  return unless fifo?
226
248
 
227
- options[:message_group_id] ||= MESSAGE_GROUP_ID
228
- options[:message_deduplication_id] ||= Digest::SHA256.hexdigest(options[:message_body].to_s)
249
+ options[:message_group_id] ||= MESSAGE_GROUP_ID
250
+
251
+ # Auto-generate a content-based dedup id unless disabled. With it on, two
252
+ # sends of an identical body within SQS's 5-minute window are deduplicated
253
+ # (the second is silently dropped); disable it to send such messages
254
+ # distinctly (requires an explicit message_deduplication_id or the queue's
255
+ # ContentBasedDeduplication attribute).
256
+ if Shoryuken.fifo_message_deduplication?
257
+ options[:message_deduplication_id] ||= Digest::SHA256.hexdigest(options[:message_body].to_s)
258
+ end
229
259
 
230
260
  options
231
261
  end
@@ -2,5 +2,5 @@
2
2
 
3
3
  module Shoryuken
4
4
  # Current version of the Shoryuken gem
5
- VERSION = '7.0.3'
5
+ VERSION = '7.0.4'
6
6
  end
@@ -15,11 +15,17 @@ module Shoryuken
15
15
  # @option options [String] :queue override the default queue
16
16
  # @return [Aws::SQS::Types::SendMessageResult] the send result
17
17
  def perform_async(worker_class, body, options = {})
18
- options[:message_attributes] ||= {}
19
- options[:message_attributes]['shoryuken_class'] = {
20
- string_value: worker_class.to_s,
21
- data_type: 'String'
22
- }
18
+ # Work on a copy: callers may reuse the same options hash across
19
+ # enqueues, and mutating it (deleting :queue, adding :message_body)
20
+ # would silently reroute or corrupt later jobs. Merge - rather than
21
+ # mutate - the nested :message_attributes hash for the same reason.
22
+ options = options.dup
23
+ options[:message_attributes] = (options[:message_attributes] || {}).merge(
24
+ 'shoryuken_class' => {
25
+ string_value: worker_class.to_s,
26
+ data_type: 'String'
27
+ }
28
+ )
23
29
 
24
30
  options[:message_body] = body
25
31
 
@@ -15,13 +15,18 @@ module Shoryuken
15
15
  # @option options [Hash] :message_attributes custom message attributes
16
16
  # @return [Object] the result of the worker's perform method
17
17
  def perform_async(worker_class, body, options = {})
18
+ # Work on a copy so reused/shared options hashes aren't mutated across
19
+ # calls (deleting :queue would reroute later jobs to the default
20
+ # queue). Merge - rather than mutate - the nested message_attributes.
21
+ options = options.dup
18
22
  body = JSON.dump(body) if body.is_a?(Hash)
19
23
  queue_name = options.delete(:queue) || worker_class.get_shoryuken_options['queue']
20
- message_attributes = options.delete(:message_attributes) || {}
21
- message_attributes['shoryuken_class'] = {
22
- string_value: worker_class.to_s,
23
- data_type: 'String'
24
- }
24
+ message_attributes = (options.delete(:message_attributes) || {}).merge(
25
+ 'shoryuken_class' => {
26
+ string_value: worker_class.to_s,
27
+ data_type: 'String'
28
+ }
29
+ )
25
30
 
26
31
  sqs_msg = InlineMessage.new(
27
32
  body: body,