shoryuken 7.0.3 → 7.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/push.yml +3 -3
- data/.github/workflows/specs.yml +4 -4
- data/.github/workflows/verify-action-pins.yml +1 -1
- data/.ruby-version +1 -1
- data/.yard-lint.yml +29 -1
- data/CHANGELOG.md +65 -1
- data/Gemfile.lint.lock +8 -7
- data/bin/cli/sqs.rb +60 -3
- data/lib/active_job/extensions.rb +5 -8
- data/lib/active_job/queue_adapters/shoryuken_adapter.rb +3 -4
- data/lib/shoryuken/active_job/current_attributes.rb +4 -6
- data/lib/shoryuken/body_parser.rb +1 -2
- data/lib/shoryuken/client.rb +15 -6
- data/lib/shoryuken/default_exception_handler.rb +2 -3
- data/lib/shoryuken/errors.rb +2 -4
- data/lib/shoryuken/helpers/atomic_counter.rb +6 -9
- data/lib/shoryuken/helpers/atomic_hash.rb +9 -15
- data/lib/shoryuken/helpers/hash_utils.rb +5 -7
- data/lib/shoryuken/helpers/string_utils.rb +6 -8
- data/lib/shoryuken/inline_message.rb +3 -4
- data/lib/shoryuken/launcher.rb +8 -13
- data/lib/shoryuken/manager.rb +27 -1
- data/lib/shoryuken/message.rb +2 -3
- data/lib/shoryuken/middleware/chain.rb +7 -13
- data/lib/shoryuken/middleware/entry.rb +1 -2
- data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +3 -5
- data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +38 -21
- data/lib/shoryuken/middleware/server/timing.rb +2 -3
- data/lib/shoryuken/options.rb +28 -6
- data/lib/shoryuken/polling/base_strategy.rb +16 -23
- data/lib/shoryuken/polling/queue_configuration.rb +8 -12
- data/lib/shoryuken/polling/weighted_round_robin.rb +3 -4
- data/lib/shoryuken/queue.rb +33 -3
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker/default_executor.rb +11 -5
- data/lib/shoryuken/worker/inline_executor.rb +10 -5
- data/lib/shoryuken/worker.rb +10 -12
- data/lib/shoryuken.rb +4 -3
- data/shoryuken.gemspec +9 -0
- data/spec/integration/cli/find_all_spec.rb +120 -0
- data/spec/lib/shoryuken/client_spec.rb +62 -0
- data/spec/lib/shoryuken/manager_spec.rb +48 -0
- data/spec/lib/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +32 -0
- data/spec/lib/shoryuken/queue_spec.rb +86 -0
- data/spec/lib/shoryuken/worker/default_executor_spec.rb +13 -0
- data/spec/lib/shoryuken/worker/inline_executor_spec.rb +12 -0
- data/spec/spec_helper.rb +14 -13
- metadata +11 -3
data/lib/shoryuken/launcher.rb
CHANGED
|
@@ -14,9 +14,8 @@ module Shoryuken
|
|
|
14
14
|
|
|
15
15
|
# Indicates whether the launcher is in the process of stopping.
|
|
16
16
|
#
|
|
17
|
-
# This flag is set to true when either {#stop} or {#stop!} is called,
|
|
18
|
-
#
|
|
19
|
-
# checkpoint and prepare for graceful shutdown.
|
|
17
|
+
# This flag is set to true when either {#stop} or {#stop!} is called, and is used by ActiveJob adapters to signal jobs
|
|
18
|
+
# that they should checkpoint and prepare for graceful shutdown.
|
|
20
19
|
#
|
|
21
20
|
# @return [Boolean] true if stopping, false otherwise
|
|
22
21
|
def stopping?
|
|
@@ -91,11 +90,9 @@ module Shoryuken
|
|
|
91
90
|
@managers.each(&:await_dispatching_in_progress)
|
|
92
91
|
end
|
|
93
92
|
|
|
94
|
-
# Shuts the executor down, giving in-flight workers up to the configured
|
|
95
|
-
#
|
|
96
|
-
#
|
|
97
|
-
# a graceful stop still waits for workers, but must not block forever on a
|
|
98
|
-
# hung one.
|
|
93
|
+
# Shuts the executor down, giving in-flight workers up to the configured timeout to finish before force-killing them so
|
|
94
|
+
# the process can exit. Used by both the graceful ({#stop}) and immediate ({#stop!}) shutdowns: a graceful stop still
|
|
95
|
+
# waits for workers, but must not block forever on a hung one.
|
|
99
96
|
#
|
|
100
97
|
# @return [void]
|
|
101
98
|
def shutdown_executor
|
|
@@ -105,11 +102,9 @@ module Shoryuken
|
|
|
105
102
|
|
|
106
103
|
# Returns the executor for running async operations
|
|
107
104
|
#
|
|
108
|
-
# Owns a dedicated executor rather than borrowing Concurrent.global_io_executor:
|
|
109
|
-
#
|
|
110
|
-
#
|
|
111
|
-
# Shoryuken's own ShoryukenConcurrentSendAdapter) and prevent a fresh launcher
|
|
112
|
-
# from starting in the same process.
|
|
105
|
+
# Owns a dedicated executor rather than borrowing Concurrent.global_io_executor: {#stop} and {#stop!} shut down and kill
|
|
106
|
+
# this executor, and destroying the process-global pool would break anything else relying on it (including Shoryuken's
|
|
107
|
+
# own ShoryukenConcurrentSendAdapter) and prevent a fresh launcher from starting in the same process.
|
|
113
108
|
#
|
|
114
109
|
# @return [Concurrent::ExecutorService] the executor service
|
|
115
110
|
def executor
|
data/lib/shoryuken/manager.rb
CHANGED
|
@@ -32,6 +32,7 @@ module Shoryuken
|
|
|
32
32
|
@executor = executor
|
|
33
33
|
@running = Shoryuken::Helpers::AtomicBoolean.new(true)
|
|
34
34
|
@stop_new_dispatching = Shoryuken::Helpers::AtomicBoolean.new(false)
|
|
35
|
+
@dispatch_started = Shoryuken::Helpers::AtomicBoolean.new(false)
|
|
35
36
|
@dispatching_release_signal = ::Queue.new
|
|
36
37
|
end
|
|
37
38
|
|
|
@@ -48,6 +49,14 @@ module Shoryuken
|
|
|
48
49
|
# @return [void]
|
|
49
50
|
def stop_new_dispatching
|
|
50
51
|
@stop_new_dispatching.make_true
|
|
52
|
+
|
|
53
|
+
# If the dispatch loop never ran (e.g. a stop arrives before the start
|
|
54
|
+
# Future is scheduled, or in an embedded host whose executor is
|
|
55
|
+
# saturated), there is no loop to observe the flag and close the release
|
|
56
|
+
# signal, so await_dispatching_in_progress would block forever. Close it
|
|
57
|
+
# here in that case. Queue#close is idempotent, so a dispatch_loop that
|
|
58
|
+
# does start later and closes it again is harmless.
|
|
59
|
+
@dispatching_release_signal.close unless @dispatch_started.true?
|
|
51
60
|
end
|
|
52
61
|
|
|
53
62
|
# Waits for any in-progress dispatching to complete
|
|
@@ -76,6 +85,11 @@ module Shoryuken
|
|
|
76
85
|
#
|
|
77
86
|
# @return [void]
|
|
78
87
|
def dispatch_loop
|
|
88
|
+
# Mark that the loop has run at least once so stop_new_dispatching knows
|
|
89
|
+
# it can rely on the loop to close the release signal (and otherwise
|
|
90
|
+
# closes it itself to avoid a deadlock).
|
|
91
|
+
@dispatch_started.make_true
|
|
92
|
+
|
|
79
93
|
if @stop_new_dispatching.true? || !running?
|
|
80
94
|
# Close (instead of push) so every pending and future
|
|
81
95
|
# await_dispatching_in_progress call returns, not just the first one
|
|
@@ -156,7 +170,8 @@ module Shoryuken
|
|
|
156
170
|
#
|
|
157
171
|
# @param queue_name [String] the queue name
|
|
158
172
|
# @param sqs_msg [Aws::SQS::Types::Message, Array<Aws::SQS::Types::Message>] the message or batch
|
|
159
|
-
# @return [Concurrent::Promise, nil] the processing promise or nil if
|
|
173
|
+
# @return [Concurrent::Promise, nil] the processing promise, or nil if the manager
|
|
174
|
+
# is not running or the executor rejects the worker post
|
|
160
175
|
def assign(queue_name, sqs_msg)
|
|
161
176
|
return unless running?
|
|
162
177
|
|
|
@@ -181,6 +196,17 @@ module Shoryuken
|
|
|
181
196
|
processor_done(queue_name)
|
|
182
197
|
end
|
|
183
198
|
end
|
|
199
|
+
rescue Concurrent::RejectedExecutionError
|
|
200
|
+
# The executor was shut down (or a bounded custom launcher_executor is
|
|
201
|
+
# saturated) between the running? check above and the post. The promise
|
|
202
|
+
# body - and therefore processor_done - never ran, so roll back the
|
|
203
|
+
# increment here. Leaking it would permanently shrink `ready`
|
|
204
|
+
# (@max_processors - busy) until dispatch stalls and the group stops
|
|
205
|
+
# processing. The message was never processed, so we must not run the
|
|
206
|
+
# FIFO message_processed callback - decrement directly instead.
|
|
207
|
+
@busy_processors.decrement
|
|
208
|
+
fire_utilization_update_event
|
|
209
|
+
nil
|
|
184
210
|
end
|
|
185
211
|
|
|
186
212
|
# Dispatches a batch of messages from a queue
|
data/lib/shoryuken/message.rb
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Shoryuken
|
|
4
|
-
# Represents an SQS message received by a Shoryuken worker.
|
|
5
|
-
#
|
|
6
|
-
# for interacting with the message, including deletion and visibility timeout management.
|
|
4
|
+
# Represents an SQS message received by a Shoryuken worker. This class wraps the raw AWS SQS message data and provides
|
|
5
|
+
# convenient methods for interacting with the message, including deletion and visibility timeout management.
|
|
7
6
|
#
|
|
8
7
|
# Message instances are automatically created by Shoryuken and passed to your
|
|
9
8
|
# worker's `perform` method as the first argument.
|
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Shoryuken
|
|
4
|
-
# Middleware provides a way to wrap message processing with custom logic,
|
|
5
|
-
#
|
|
6
|
-
# side and can perform setup, teardown, error handling, and monitoring around
|
|
7
|
-
# job execution.
|
|
4
|
+
# Middleware provides a way to wrap message processing with custom logic, similar to Rack middleware in web applications.
|
|
5
|
+
# Middleware runs on the server side and can perform setup, teardown, error handling, and monitoring around job execution.
|
|
8
6
|
#
|
|
9
7
|
# Middleware classes must implement a `call` method that accepts the worker instance,
|
|
10
8
|
# queue name, and SQS message, and must yield to continue the middleware chain.
|
|
@@ -84,9 +82,8 @@ module Shoryuken
|
|
|
84
82
|
# @see Shoryuken::Middleware::Chain Middleware chain management
|
|
85
83
|
# @see https://github.com/ruby-shoryuken/shoryuken/wiki/Middleware Comprehensive middleware guide
|
|
86
84
|
module Middleware
|
|
87
|
-
# Manages a chain of middleware classes that will be instantiated and invoked
|
|
88
|
-
#
|
|
89
|
-
# and reordering middleware.
|
|
85
|
+
# Manages a chain of middleware classes that will be instantiated and invoked in sequence around message processing.
|
|
86
|
+
# Provides methods for adding, removing, and reordering middleware.
|
|
90
87
|
class Chain
|
|
91
88
|
# @return [Array<Entry>] The ordered list of middleware entries
|
|
92
89
|
attr_reader :entries
|
|
@@ -121,8 +118,7 @@ module Shoryuken
|
|
|
121
118
|
entries.delete_if { |entry| entry.klass == klass }
|
|
122
119
|
end
|
|
123
120
|
|
|
124
|
-
# Adds middleware to the end of the chain. Does nothing if the middleware
|
|
125
|
-
# class is already present in the chain.
|
|
121
|
+
# Adds middleware to the end of the chain. Does nothing if the middleware class is already present in the chain.
|
|
126
122
|
#
|
|
127
123
|
# @param klass [Class] The middleware class to add
|
|
128
124
|
# @param args [Array] Arguments to pass to the middleware constructor
|
|
@@ -132,8 +128,7 @@ module Shoryuken
|
|
|
132
128
|
entries << Entry.new(klass, *args) unless exists?(klass)
|
|
133
129
|
end
|
|
134
130
|
|
|
135
|
-
# Adds middleware to the beginning of the chain. Does nothing if the middleware
|
|
136
|
-
# class is already present in the chain.
|
|
131
|
+
# Adds middleware to the beginning of the chain. Does nothing if the middleware class is already present in the chain.
|
|
137
132
|
#
|
|
138
133
|
# @param klass [Class] The middleware class to prepend
|
|
139
134
|
# @param args [Array] Arguments to pass to the middleware constructor
|
|
@@ -195,8 +190,7 @@ module Shoryuken
|
|
|
195
190
|
entries.clear
|
|
196
191
|
end
|
|
197
192
|
|
|
198
|
-
# Invokes the middleware chain with the given arguments.
|
|
199
|
-
# Each middleware's call method will be invoked in sequence,
|
|
193
|
+
# Invokes the middleware chain with the given arguments. Each middleware's call method will be invoked in sequence,
|
|
200
194
|
# with control passed through yielding.
|
|
201
195
|
#
|
|
202
196
|
# @param args [Array] arguments to pass to each middleware
|
|
@@ -2,8 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Shoryuken
|
|
4
4
|
module Middleware
|
|
5
|
-
# Represents an entry in a middleware chain, storing the middleware class
|
|
6
|
-
# and any arguments needed for its instantiation.
|
|
5
|
+
# Represents an entry in a middleware chain, storing the middleware class and any arguments needed for its instantiation.
|
|
7
6
|
#
|
|
8
7
|
# @api private
|
|
9
8
|
class Entry
|
|
@@ -73,11 +73,9 @@ module Shoryuken
|
|
|
73
73
|
|
|
74
74
|
private
|
|
75
75
|
|
|
76
|
-
# Returns a positive interval at which to re-extend the message's
|
|
77
|
-
# visibility
|
|
78
|
-
#
|
|
79
|
-
# before expiry, but for short timeouts (<= EXTEND_UPFRONT_SECONDS) it
|
|
80
|
-
# falls back to half the timeout so the timer still fires in time
|
|
76
|
+
# Returns a positive interval at which to re-extend the message's visibility before it expires, or nil when the
|
|
77
|
+
# visibility timeout is too short to schedule one. Normally this is EXTEND_UPFRONT_SECONDS before expiry, but for
|
|
78
|
+
# short timeouts (<= EXTEND_UPFRONT_SECONDS) it falls back to half the timeout so the timer still fires in time
|
|
81
79
|
# instead of TimerTask raising on a non-positive interval.
|
|
82
80
|
#
|
|
83
81
|
# @param visibility_timeout [Integer] the queue's visibility timeout in seconds
|
|
@@ -3,9 +3,8 @@
|
|
|
3
3
|
module Shoryuken
|
|
4
4
|
module Middleware
|
|
5
5
|
module Server
|
|
6
|
-
# Middleware that implements exponential backoff retry for failed messages.
|
|
7
|
-
#
|
|
8
|
-
# configured retry intervals.
|
|
6
|
+
# Middleware that implements exponential backoff retry for failed messages. When a job fails, the message visibility
|
|
7
|
+
# timeout is adjusted based on configured retry intervals.
|
|
9
8
|
class ExponentialBackoffRetry
|
|
10
9
|
include Util
|
|
11
10
|
|
|
@@ -28,24 +27,31 @@ module Shoryuken
|
|
|
28
27
|
end
|
|
29
28
|
|
|
30
29
|
started_at = Time.now
|
|
31
|
-
yield
|
|
32
|
-
rescue => e
|
|
33
|
-
worker_options = worker.class.get_shoryuken_options
|
|
34
|
-
retry_intervals = worker_options['retry_intervals']
|
|
35
|
-
|
|
36
|
-
# Non-retryable exceptions must not be backoff retried; re-raise so the
|
|
37
|
-
# NonRetryableException middleware can delete the message immediately.
|
|
38
|
-
raise if NonRetryableException.non_retryable?(e, worker_options['non_retryable_exceptions'])
|
|
39
30
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
31
|
+
# Scope the rescue to the single-message yield only. A method-level
|
|
32
|
+
# rescue would also catch the batch `return yield` above, then route the
|
|
33
|
+
# Array into handle_failure (which calls #attributes/#message_id on it),
|
|
34
|
+
# raising a NoMethodError that masks the original worker error.
|
|
35
|
+
begin
|
|
36
|
+
yield
|
|
37
|
+
rescue => e
|
|
38
|
+
worker_options = worker.class.get_shoryuken_options
|
|
39
|
+
retry_intervals = worker_options['retry_intervals']
|
|
40
|
+
|
|
41
|
+
# Non-retryable exceptions must not be backoff retried; re-raise so the
|
|
42
|
+
# NonRetryableException middleware can delete the message immediately.
|
|
43
|
+
raise if NonRetryableException.non_retryable?(e, worker_options['non_retryable_exceptions'])
|
|
44
|
+
|
|
45
|
+
if retry_intervals.nil? || !handle_failure(sqs_msg, started_at, retry_intervals)
|
|
46
|
+
# Re-raise the exception if the job is not going to be exponential backoff retried.
|
|
47
|
+
# This allows custom middleware (like exception notifiers) to be aware of the unhandled failure.
|
|
48
|
+
raise
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
logger.warn { "Message #{sqs_msg.message_id} will attempt retry due to error: #{e.message}" }
|
|
52
|
+
# since we didn't raise, lets log the backtrace for debugging purposes.
|
|
53
|
+
logger.debug { e.backtrace.join("\n") } unless e.backtrace.nil?
|
|
44
54
|
end
|
|
45
|
-
|
|
46
|
-
logger.warn { "Message #{sqs_msg.message_id} will attempt retry due to error: #{e.message}" }
|
|
47
|
-
# since we didn't raise, lets log the backtrace for debugging purposes.
|
|
48
|
-
logger.debug { e.backtrace.join("\n") } unless e.backtrace.nil?
|
|
49
55
|
end
|
|
50
56
|
|
|
51
57
|
private
|
|
@@ -69,11 +75,14 @@ module Shoryuken
|
|
|
69
75
|
#
|
|
70
76
|
# @param interval [Integer] the desired interval
|
|
71
77
|
# @param started_at [Time] when processing started
|
|
72
|
-
# @return [Integer] the capped visibility timeout
|
|
78
|
+
# @return [Integer] the capped visibility timeout (never negative)
|
|
73
79
|
def next_visibility_timeout(interval, started_at)
|
|
74
80
|
max_timeout = 43_200 - (Time.now - started_at).ceil - 1
|
|
75
81
|
interval = max_timeout if interval > max_timeout
|
|
76
|
-
|
|
82
|
+
# Never go negative: a job that ran longer than the 12h SQS ceiling
|
|
83
|
+
# drives max_timeout below zero, and change_message_visibility rejects
|
|
84
|
+
# a negative timeout. Clamp to 0 (retry as soon as possible) instead.
|
|
85
|
+
[interval.to_i, 0].max
|
|
77
86
|
end
|
|
78
87
|
|
|
79
88
|
# Handles a message failure by adjusting visibility timeout
|
|
@@ -92,6 +101,14 @@ module Shoryuken
|
|
|
92
101
|
logger.info { "Message #{sqs_msg.message_id} failed, will be retried in #{interval} seconds" }
|
|
93
102
|
|
|
94
103
|
true
|
|
104
|
+
rescue => e
|
|
105
|
+
# Rescheduling itself failed (e.g. an expired receipt handle, or a
|
|
106
|
+
# transient SQS error). Don't let that exception escape and mask the
|
|
107
|
+
# original worker failure - log it and report "not retried" so the
|
|
108
|
+
# caller re-raises the original error and the message falls back to the
|
|
109
|
+
# queue's default visibility timeout.
|
|
110
|
+
logger.warn { "Failed to set backoff visibility timeout for #{sqs_msg.message_id}: #{e.message}" }
|
|
111
|
+
false
|
|
95
112
|
end
|
|
96
113
|
end
|
|
97
114
|
end
|
|
@@ -3,9 +3,8 @@
|
|
|
3
3
|
module Shoryuken
|
|
4
4
|
module Middleware
|
|
5
5
|
module Server
|
|
6
|
-
# Middleware that logs timing information for message processing.
|
|
7
|
-
#
|
|
8
|
-
# exceeds the queue's visibility timeout.
|
|
6
|
+
# Middleware that logs timing information for message processing. Records start time, completion time, and warns if
|
|
7
|
+
# processing exceeds the queue's visibility timeout.
|
|
9
8
|
class Timing
|
|
10
9
|
include Util
|
|
11
10
|
|
data/lib/shoryuken/options.rb
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Shoryuken
|
|
4
|
-
# Stores and manages all Shoryuken configuration options.
|
|
5
|
-
#
|
|
6
|
-
# middleware, and other runtime configurations.
|
|
4
|
+
# Stores and manages all Shoryuken configuration options. This class is used internally to hold settings for workers,
|
|
5
|
+
# queues, middleware, and other runtime configurations.
|
|
7
6
|
class Options
|
|
8
7
|
# Default configuration values for Shoryuken
|
|
9
8
|
DEFAULTS = {
|
|
@@ -35,6 +34,7 @@ module Shoryuken
|
|
|
35
34
|
# @return [Shoryuken::WorkerRegistry] the registry for worker classes
|
|
36
35
|
# @return [Array<#call>] handlers for processing exceptions
|
|
37
36
|
attr_accessor :active_job_queue_name_prefixing, :active_job_fifo_message_deduplication,
|
|
37
|
+
:fifo_message_deduplication,
|
|
38
38
|
:cache_visibility_timeout,
|
|
39
39
|
:groups, :launcher_executor, :reloader, :enable_reloading,
|
|
40
40
|
:start_callback, :stop_callback, :worker_executor, :worker_registry,
|
|
@@ -55,6 +55,7 @@ module Shoryuken
|
|
|
55
55
|
self.exception_handlers = [DefaultExceptionHandler]
|
|
56
56
|
self.active_job_queue_name_prefixing = false
|
|
57
57
|
self.active_job_fifo_message_deduplication = true
|
|
58
|
+
self.fifo_message_deduplication = true
|
|
58
59
|
self.worker_executor = Worker::DefaultExecutor
|
|
59
60
|
self.cache_visibility_timeout = false
|
|
60
61
|
self.reloader = proc { |&block| block.call }
|
|
@@ -304,15 +305,36 @@ module Shoryuken
|
|
|
304
305
|
@active_job_queue_name_prefixing
|
|
305
306
|
end
|
|
306
307
|
|
|
307
|
-
# Checks if the ActiveJob adapter should auto-generate a content-based
|
|
308
|
-
#
|
|
309
|
-
# of the same job class and arguments are no longer silently deduplicated.
|
|
308
|
+
# Checks if the ActiveJob adapter should auto-generate a content-based message_deduplication_id for FIFO queues. When
|
|
309
|
+
# disabled, distinct enqueues of the same job class and arguments are no longer silently deduplicated.
|
|
310
310
|
#
|
|
311
311
|
# @return [Boolean] true if FIFO deduplication id generation is enabled
|
|
312
312
|
def active_job_fifo_message_deduplication?
|
|
313
313
|
@active_job_fifo_message_deduplication
|
|
314
314
|
end
|
|
315
315
|
|
|
316
|
+
# Checks whether Shoryuken auto-generates a content-based
|
|
317
|
+
# message_deduplication_id for raw FIFO sends (Worker.perform_async,
|
|
318
|
+
# Queue#send_message / #send_messages). When disabled, distinct sends of an
|
|
319
|
+
# identical body are no longer silently deduplicated by SQS - you must then
|
|
320
|
+
# provide a message_deduplication_id yourself or enable
|
|
321
|
+
# ContentBasedDeduplication on the queue.
|
|
322
|
+
#
|
|
323
|
+
# ActiveJob enqueues are primarily controlled by
|
|
324
|
+
# #active_job_fifo_message_deduplication?. When that is enabled the adapter
|
|
325
|
+
# sets a content-based message_deduplication_id itself (from the body minus
|
|
326
|
+
# job_id/enqueued_at) before the raw send, so this flag is a no-op for them.
|
|
327
|
+
# When it is disabled the adapter omits the id and this flag governs the
|
|
328
|
+
# fallback: left enabled (the default) each enqueue still gets a hash of the
|
|
329
|
+
# full serialized body - which includes the per-enqueue job_id, so distinct
|
|
330
|
+
# enqueues are NOT deduplicated - while disabling it drops the id entirely
|
|
331
|
+
# (then requiring an explicit id or the queue's ContentBasedDeduplication).
|
|
332
|
+
#
|
|
333
|
+
# @return [Boolean] true if FIFO deduplication id generation is enabled
|
|
334
|
+
def fifo_message_deduplication?
|
|
335
|
+
@fifo_message_deduplication
|
|
336
|
+
end
|
|
337
|
+
|
|
316
338
|
private
|
|
317
339
|
|
|
318
340
|
# Creates the default server middleware chain
|
|
@@ -5,10 +5,9 @@ module Shoryuken
|
|
|
5
5
|
module Polling
|
|
6
6
|
# Abstract base class for queue polling strategies.
|
|
7
7
|
#
|
|
8
|
-
# This class defines the interface that all polling strategies must implement
|
|
9
|
-
#
|
|
10
|
-
#
|
|
11
|
-
# how to handle scenarios where queues have no messages available.
|
|
8
|
+
# This class defines the interface that all polling strategies must implement to manage queue selection and message flow
|
|
9
|
+
# control in Shoryuken workers. Polling strategies determine which queue to fetch messages from next and how to handle
|
|
10
|
+
# scenarios where queues have no messages available.
|
|
12
11
|
#
|
|
13
12
|
# @abstract Subclass and override {#next_queue}, {#messages_found}, and {#active_queues}
|
|
14
13
|
# to implement a custom polling strategy.
|
|
@@ -42,9 +41,8 @@ module Shoryuken
|
|
|
42
41
|
|
|
43
42
|
# Returns the next queue to poll for messages.
|
|
44
43
|
#
|
|
45
|
-
# This method should return a QueueConfiguration object representing
|
|
46
|
-
#
|
|
47
|
-
# queues are currently available for polling.
|
|
44
|
+
# This method should return a QueueConfiguration object representing the next queue that should be polled for messages,
|
|
45
|
+
# or nil if no queues are currently available for polling.
|
|
48
46
|
#
|
|
49
47
|
# @abstract Subclasses must implement this method
|
|
50
48
|
# @return [QueueConfiguration, nil] Next queue to poll, or nil if none available
|
|
@@ -55,10 +53,9 @@ module Shoryuken
|
|
|
55
53
|
|
|
56
54
|
# Called when messages are found (or not found) in a queue.
|
|
57
55
|
#
|
|
58
|
-
# This method is invoked after polling a queue to inform the strategy
|
|
59
|
-
# about
|
|
60
|
-
#
|
|
61
|
-
# such as pausing empty queues or adjusting queue weights.
|
|
56
|
+
# This method is invoked after polling a queue to inform the strategy about the number of messages that were retrieved.
|
|
57
|
+
# Strategies can use this information to make decisions about future polling behavior, such as pausing empty queues or
|
|
58
|
+
# adjusting queue weights.
|
|
62
59
|
#
|
|
63
60
|
# @abstract Subclasses must implement this method
|
|
64
61
|
# @param _queue [String] The name of the queue that was polled
|
|
@@ -70,9 +67,8 @@ module Shoryuken
|
|
|
70
67
|
|
|
71
68
|
# Called when a message from a queue has been processed.
|
|
72
69
|
#
|
|
73
|
-
# This optional callback is invoked after a message has been successfully
|
|
74
|
-
#
|
|
75
|
-
# or to adjust their polling behavior.
|
|
70
|
+
# This optional callback is invoked after a message has been successfully processed by a worker. Strategies can use
|
|
71
|
+
# this information for cleanup or to adjust their polling behavior.
|
|
76
72
|
#
|
|
77
73
|
# @param _queue [String] The name of the queue whose message was processed
|
|
78
74
|
# @return [void]
|
|
@@ -80,9 +76,8 @@ module Shoryuken
|
|
|
80
76
|
|
|
81
77
|
# Returns the list of currently active queues.
|
|
82
78
|
#
|
|
83
|
-
# This method should return an array representing the queues that are
|
|
84
|
-
#
|
|
85
|
-
# strategy implementation.
|
|
79
|
+
# This method should return an array representing the queues that are currently active and available for polling. The
|
|
80
|
+
# format may vary by strategy implementation.
|
|
86
81
|
#
|
|
87
82
|
# @abstract Subclasses must implement this method
|
|
88
83
|
# @return [Array] List of active queues
|
|
@@ -93,9 +88,8 @@ module Shoryuken
|
|
|
93
88
|
|
|
94
89
|
# Compares this strategy with another object for equality.
|
|
95
90
|
#
|
|
96
|
-
# Two strategies are considered equal if they have the same active queues.
|
|
97
|
-
#
|
|
98
|
-
# compatibility.
|
|
91
|
+
# Two strategies are considered equal if they have the same active queues. This method also supports comparison with
|
|
92
|
+
# Array objects for backward compatibility.
|
|
99
93
|
#
|
|
100
94
|
# @param other [Object] Object to compare with
|
|
101
95
|
# @return [Boolean] true if strategies are equivalent
|
|
@@ -114,9 +108,8 @@ module Shoryuken
|
|
|
114
108
|
|
|
115
109
|
# Returns the delay time for pausing empty queues.
|
|
116
110
|
#
|
|
117
|
-
# This method returns the amount of time (in seconds) that empty queues
|
|
118
|
-
#
|
|
119
|
-
# the strategy level or falls back to the global Shoryuken delay setting.
|
|
111
|
+
# This method returns the amount of time (in seconds) that empty queues should be paused before being polled again. The
|
|
112
|
+
# delay can be set at the strategy level or falls back to the global Shoryuken delay setting.
|
|
120
113
|
#
|
|
121
114
|
# @return [Float] Delay time in seconds
|
|
122
115
|
def delay
|
|
@@ -4,9 +4,8 @@ module Shoryuken
|
|
|
4
4
|
module Polling
|
|
5
5
|
# Configuration object representing a queue and its associated options.
|
|
6
6
|
#
|
|
7
|
-
# This class encapsulates a queue name along with any polling-specific
|
|
8
|
-
#
|
|
9
|
-
# information between polling strategies and the message fetching system.
|
|
7
|
+
# This class encapsulates a queue name along with any polling-specific options or metadata. It provides a structured way
|
|
8
|
+
# to pass queue information between polling strategies and the message fetching system.
|
|
10
9
|
#
|
|
11
10
|
# The class extends Struct to provide attribute accessors for name and options
|
|
12
11
|
# while adding custom behavior for equality comparison and string representation.
|
|
@@ -32,9 +31,8 @@ module Shoryuken
|
|
|
32
31
|
QueueConfiguration = Struct.new(:name, :options) do
|
|
33
32
|
# Generates a hash value based on the queue name.
|
|
34
33
|
#
|
|
35
|
-
# This method ensures that QueueConfiguration objects can be used
|
|
36
|
-
#
|
|
37
|
-
# will have the same hash value regardless of their options.
|
|
34
|
+
# This method ensures that QueueConfiguration objects can be used as hash keys and that configurations with the same
|
|
35
|
+
# queue name will have the same hash value regardless of their options.
|
|
38
36
|
#
|
|
39
37
|
# @return [Integer] Hash value based on the queue name
|
|
40
38
|
def hash
|
|
@@ -43,9 +41,8 @@ module Shoryuken
|
|
|
43
41
|
|
|
44
42
|
# Compares this configuration with another object for equality.
|
|
45
43
|
#
|
|
46
|
-
# Two QueueConfiguration objects are equal if they have the same name
|
|
47
|
-
#
|
|
48
|
-
# also be compared directly with a string queue name.
|
|
44
|
+
# Two QueueConfiguration objects are equal if they have the same name and options. For convenience, a configuration
|
|
45
|
+
# with empty options can also be compared directly with a string queue name.
|
|
49
46
|
#
|
|
50
47
|
# @param other [Object] The object to compare with
|
|
51
48
|
# @return [Boolean] true if the objects are considered equal
|
|
@@ -78,9 +75,8 @@ module Shoryuken
|
|
|
78
75
|
|
|
79
76
|
# Returns a string representation of the queue configuration.
|
|
80
77
|
#
|
|
81
|
-
# For configurations with empty options, returns just the queue name.
|
|
82
|
-
#
|
|
83
|
-
# showing both the name and the options hash.
|
|
78
|
+
# For configurations with empty options, returns just the queue name. For configurations with options, returns a
|
|
79
|
+
# detailed representation showing both the name and the options hash.
|
|
84
80
|
#
|
|
85
81
|
# @return [String] String representation of the configuration
|
|
86
82
|
#
|
|
@@ -90,10 +90,9 @@ module Shoryuken
|
|
|
90
90
|
|
|
91
91
|
# Unpauses the first queue whose delay has expired.
|
|
92
92
|
#
|
|
93
|
-
# Scans for any expired entry rather than only the head of the list:
|
|
94
|
-
#
|
|
95
|
-
#
|
|
96
|
-
# paused - checking only the head would leave the ready queue stuck.
|
|
93
|
+
# Scans for any expired entry rather than only the head of the list: message_processed marks a queue ready by setting
|
|
94
|
+
# its time to the epoch, and that entry may sit behind an earlier-paused queue that is still paused - checking only the
|
|
95
|
+
# head would leave the ready queue stuck.
|
|
97
96
|
#
|
|
98
97
|
# @return [void]
|
|
99
98
|
def unpause_queues
|
data/lib/shoryuken/queue.rb
CHANGED
|
@@ -89,7 +89,11 @@ module Shoryuken
|
|
|
89
89
|
# @option options [Array<Hash>] :entries message entries to send
|
|
90
90
|
# @return [Aws::SQS::Types::SendMessageBatchResult] the batch send result
|
|
91
91
|
def send_messages(options)
|
|
92
|
-
client.send_message_batch(sanitize_messages!(options).merge(queue_url: url))
|
|
92
|
+
response = client.send_message_batch(sanitize_messages!(options).merge(queue_url: url))
|
|
93
|
+
|
|
94
|
+
log_failed_sends(response)
|
|
95
|
+
|
|
96
|
+
response
|
|
93
97
|
end
|
|
94
98
|
|
|
95
99
|
# Receives messages from the queue
|
|
@@ -120,6 +124,24 @@ module Shoryuken
|
|
|
120
124
|
|
|
121
125
|
private
|
|
122
126
|
|
|
127
|
+
# Logs any per-entry failures from a batch send. SQS reports these in
|
|
128
|
+
# response.failed (throttling, oversize/malformed entries, or a same-batch
|
|
129
|
+
# duplicate message_deduplication_id) rather than raising, so without this
|
|
130
|
+
# they are silently swallowed - notably by ShoryukenAdapter#enqueue_all,
|
|
131
|
+
# which reads only response.successful. Mirrors delete_messages.
|
|
132
|
+
#
|
|
133
|
+
# @param response [Aws::SQS::Types::SendMessageBatchResult] the batch result
|
|
134
|
+
# @return [void]
|
|
135
|
+
def log_failed_sends(response)
|
|
136
|
+
return unless response.respond_to?(:failed)
|
|
137
|
+
|
|
138
|
+
Array(response.failed).each do |failure|
|
|
139
|
+
logger.error do
|
|
140
|
+
"Could not send #{failure.id}, code: '#{failure.code}', message: '#{failure.message}', sender_fault: #{failure.sender_fault}"
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
123
145
|
# Initializes the FIFO attribute by calling fifo?
|
|
124
146
|
#
|
|
125
147
|
# @return [Boolean] whether the queue is FIFO
|
|
@@ -224,8 +246,16 @@ module Shoryuken
|
|
|
224
246
|
def add_fifo_attributes!(options)
|
|
225
247
|
return unless fifo?
|
|
226
248
|
|
|
227
|
-
options[:message_group_id]
|
|
228
|
-
|
|
249
|
+
options[:message_group_id] ||= MESSAGE_GROUP_ID
|
|
250
|
+
|
|
251
|
+
# Auto-generate a content-based dedup id unless disabled. With it on, two
|
|
252
|
+
# sends of an identical body within SQS's 5-minute window are deduplicated
|
|
253
|
+
# (the second is silently dropped); disable it to send such messages
|
|
254
|
+
# distinctly (requires an explicit message_deduplication_id or the queue's
|
|
255
|
+
# ContentBasedDeduplication attribute).
|
|
256
|
+
if Shoryuken.fifo_message_deduplication?
|
|
257
|
+
options[:message_deduplication_id] ||= Digest::SHA256.hexdigest(options[:message_body].to_s)
|
|
258
|
+
end
|
|
229
259
|
|
|
230
260
|
options
|
|
231
261
|
end
|
data/lib/shoryuken/version.rb
CHANGED
|
@@ -15,11 +15,17 @@ module Shoryuken
|
|
|
15
15
|
# @option options [String] :queue override the default queue
|
|
16
16
|
# @return [Aws::SQS::Types::SendMessageResult] the send result
|
|
17
17
|
def perform_async(worker_class, body, options = {})
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
18
|
+
# Work on a copy: callers may reuse the same options hash across
|
|
19
|
+
# enqueues, and mutating it (deleting :queue, adding :message_body)
|
|
20
|
+
# would silently reroute or corrupt later jobs. Merge - rather than
|
|
21
|
+
# mutate - the nested :message_attributes hash for the same reason.
|
|
22
|
+
options = options.dup
|
|
23
|
+
options[:message_attributes] = (options[:message_attributes] || {}).merge(
|
|
24
|
+
'shoryuken_class' => {
|
|
25
|
+
string_value: worker_class.to_s,
|
|
26
|
+
data_type: 'String'
|
|
27
|
+
}
|
|
28
|
+
)
|
|
23
29
|
|
|
24
30
|
options[:message_body] = body
|
|
25
31
|
|
|
@@ -15,13 +15,18 @@ module Shoryuken
|
|
|
15
15
|
# @option options [Hash] :message_attributes custom message attributes
|
|
16
16
|
# @return [Object] the result of the worker's perform method
|
|
17
17
|
def perform_async(worker_class, body, options = {})
|
|
18
|
+
# Work on a copy so reused/shared options hashes aren't mutated across
|
|
19
|
+
# calls (deleting :queue would reroute later jobs to the default
|
|
20
|
+
# queue). Merge - rather than mutate - the nested message_attributes.
|
|
21
|
+
options = options.dup
|
|
18
22
|
body = JSON.dump(body) if body.is_a?(Hash)
|
|
19
23
|
queue_name = options.delete(:queue) || worker_class.get_shoryuken_options['queue']
|
|
20
|
-
message_attributes = options.delete(:message_attributes) || {}
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
24
|
+
message_attributes = (options.delete(:message_attributes) || {}).merge(
|
|
25
|
+
'shoryuken_class' => {
|
|
26
|
+
string_value: worker_class.to_s,
|
|
27
|
+
data_type: 'String'
|
|
28
|
+
}
|
|
29
|
+
)
|
|
25
30
|
|
|
26
31
|
sqs_msg = InlineMessage.new(
|
|
27
32
|
body: body,
|