shoryuken 7.0.3 → 7.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/push.yml +3 -3
- data/.github/workflows/specs.yml +4 -4
- data/.github/workflows/verify-action-pins.yml +1 -1
- data/.ruby-version +1 -1
- data/.yard-lint.yml +29 -1
- data/CHANGELOG.md +65 -1
- data/Gemfile.lint.lock +8 -7
- data/bin/cli/sqs.rb +60 -3
- data/lib/active_job/extensions.rb +5 -8
- data/lib/active_job/queue_adapters/shoryuken_adapter.rb +3 -4
- data/lib/shoryuken/active_job/current_attributes.rb +4 -6
- data/lib/shoryuken/body_parser.rb +1 -2
- data/lib/shoryuken/client.rb +15 -6
- data/lib/shoryuken/default_exception_handler.rb +2 -3
- data/lib/shoryuken/errors.rb +2 -4
- data/lib/shoryuken/helpers/atomic_counter.rb +6 -9
- data/lib/shoryuken/helpers/atomic_hash.rb +9 -15
- data/lib/shoryuken/helpers/hash_utils.rb +5 -7
- data/lib/shoryuken/helpers/string_utils.rb +6 -8
- data/lib/shoryuken/inline_message.rb +3 -4
- data/lib/shoryuken/launcher.rb +8 -13
- data/lib/shoryuken/manager.rb +27 -1
- data/lib/shoryuken/message.rb +2 -3
- data/lib/shoryuken/middleware/chain.rb +7 -13
- data/lib/shoryuken/middleware/entry.rb +1 -2
- data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +3 -5
- data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +38 -21
- data/lib/shoryuken/middleware/server/timing.rb +2 -3
- data/lib/shoryuken/options.rb +28 -6
- data/lib/shoryuken/polling/base_strategy.rb +16 -23
- data/lib/shoryuken/polling/queue_configuration.rb +8 -12
- data/lib/shoryuken/polling/weighted_round_robin.rb +3 -4
- data/lib/shoryuken/queue.rb +33 -3
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker/default_executor.rb +11 -5
- data/lib/shoryuken/worker/inline_executor.rb +10 -5
- data/lib/shoryuken/worker.rb +10 -12
- data/lib/shoryuken.rb +4 -3
- data/shoryuken.gemspec +9 -0
- data/spec/integration/cli/find_all_spec.rb +120 -0
- data/spec/lib/shoryuken/client_spec.rb +62 -0
- data/spec/lib/shoryuken/manager_spec.rb +48 -0
- data/spec/lib/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +32 -0
- data/spec/lib/shoryuken/queue_spec.rb +86 -0
- data/spec/lib/shoryuken/worker/default_executor_spec.rb +13 -0
- data/spec/lib/shoryuken/worker/inline_executor_spec.rb +12 -0
- data/spec/spec_helper.rb +14 -13
- metadata +11 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b332f880ab52b8380e13758f98bbca5d35a5a21f693753d536564c2dc760eadb
|
|
4
|
+
data.tar.gz: 9660f222a93c1fc517cc7f65bf029ee6c31e9a6b5526d67af917a91e64768ca4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ce8200034dfe12e010bbb928f73d7ae35f2abfda4d85235cc7af189d26306d254bfd84e6956dace838ba0a222768628cf2c4785f657f8c23cd07ddc47219064a
|
|
7
|
+
data.tar.gz: c7a947da8aa8dadd9f79bedd89d02a67cfd715e715294d5e38a396cbb7070e5a06b4aa4055e347493eaa9373242593a5eacaed27ebb7528f16b21d8007a44825
|
data/.github/workflows/push.yml
CHANGED
|
@@ -19,12 +19,12 @@ jobs:
|
|
|
19
19
|
id-token: write
|
|
20
20
|
|
|
21
21
|
steps:
|
|
22
|
-
- uses: actions/checkout@
|
|
22
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
23
23
|
with:
|
|
24
24
|
fetch-depth: 0
|
|
25
25
|
|
|
26
26
|
- name: Set up Ruby
|
|
27
|
-
uses: ruby/setup-ruby@
|
|
27
|
+
uses: ruby/setup-ruby@a0102e0972be65f351c307e2d64b9314a57c8073 # v1.324.0
|
|
28
28
|
with:
|
|
29
29
|
bundler-cache: false
|
|
30
30
|
|
|
@@ -33,4 +33,4 @@ jobs:
|
|
|
33
33
|
bundle install --jobs 4 --retry 3
|
|
34
34
|
|
|
35
35
|
# Release
|
|
36
|
-
- uses: rubygems/release-gem@
|
|
36
|
+
- uses: rubygems/release-gem@7f9650160c1a4e7989fdc9855807bdbd421d8b6b # v1.4.1
|
data/.github/workflows/specs.yml
CHANGED
|
@@ -16,9 +16,9 @@ jobs:
|
|
|
16
16
|
timeout-minutes: 10
|
|
17
17
|
steps:
|
|
18
18
|
- name: Checkout code
|
|
19
|
-
uses: actions/checkout@
|
|
19
|
+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
20
20
|
|
|
21
|
-
- uses: ruby/setup-ruby@
|
|
21
|
+
- uses: ruby/setup-ruby@a0102e0972be65f351c307e2d64b9314a57c8073 # v1.324.0
|
|
22
22
|
with:
|
|
23
23
|
ruby-version: ${{ matrix.ruby }}
|
|
24
24
|
bundler-cache: true
|
|
@@ -57,7 +57,7 @@ jobs:
|
|
|
57
57
|
timeout-minutes: 15
|
|
58
58
|
steps:
|
|
59
59
|
- name: Checkout code
|
|
60
|
-
uses: actions/checkout@
|
|
60
|
+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
61
61
|
|
|
62
62
|
- name: Start ElasticMQ
|
|
63
63
|
run: docker compose up -d
|
|
@@ -71,7 +71,7 @@ jobs:
|
|
|
71
71
|
done
|
|
72
72
|
'
|
|
73
73
|
|
|
74
|
-
- uses: ruby/setup-ruby@
|
|
74
|
+
- uses: ruby/setup-ruby@a0102e0972be65f351c307e2d64b9314a57c8073 # v1.324.0
|
|
75
75
|
with:
|
|
76
76
|
ruby-version: ${{ matrix.ruby }}
|
|
77
77
|
bundler-cache: true
|
|
@@ -7,7 +7,7 @@ jobs:
|
|
|
7
7
|
verify:
|
|
8
8
|
runs-on: ubuntu-latest
|
|
9
9
|
steps:
|
|
10
|
-
- uses: actions/checkout@
|
|
10
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
11
11
|
- name: Check SHA pins
|
|
12
12
|
run: |
|
|
13
13
|
if grep -E -r "uses: .*/.*@(v[0-9]+|main|master)($|[[:space:]]|$)" --include="*.yml" --include="*.yaml" .github/workflows/ | grep -v "#"; then
|
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
4.0.
|
|
1
|
+
4.0.7
|
data/.yard-lint.yml
CHANGED
|
@@ -66,6 +66,34 @@ Documentation/BlankLineBeforeDefinition:
|
|
|
66
66
|
SingleBlankLine: true
|
|
67
67
|
OrphanedDocs: true
|
|
68
68
|
|
|
69
|
+
Documentation/DuplicateNamespaceComment:
|
|
70
|
+
Description: Detects namespaces documented with a YARD comment in more than one file.
|
|
71
|
+
Enabled: true
|
|
72
|
+
Severity: error
|
|
73
|
+
|
|
74
|
+
Documentation/UnderfilledLines:
|
|
75
|
+
Description: Detects documentation prose that wraps too early and wastes horizontal space.
|
|
76
|
+
Enabled: true
|
|
77
|
+
Severity: error
|
|
78
|
+
# Aligned with RuboCop's Layout/LineLength so documentation prose wraps to the same
|
|
79
|
+
# width as code.
|
|
80
|
+
MaxLength: 125
|
|
81
|
+
|
|
82
|
+
Documentation/LineLength:
|
|
83
|
+
Description: Detects documentation lines that exceed the maximum length.
|
|
84
|
+
Enabled: true
|
|
85
|
+
Severity: error
|
|
86
|
+
# Aligned with RuboCop's Layout/LineLength.
|
|
87
|
+
MaxLength: 125
|
|
88
|
+
|
|
89
|
+
Documentation/TextSubstitution:
|
|
90
|
+
Description: Detects em/en-dashes in documentation and replaces them with hyphens.
|
|
91
|
+
Enabled: true
|
|
92
|
+
Severity: error
|
|
93
|
+
Substitutions:
|
|
94
|
+
"—": "-" # em-dash (U+2014)
|
|
95
|
+
"–": "-" # en-dash (U+2013)
|
|
96
|
+
|
|
69
97
|
# Tags validators
|
|
70
98
|
Tags/Order:
|
|
71
99
|
Description: Enforces consistent ordering of YARD tags.
|
|
@@ -146,7 +174,7 @@ Tags/OptionTags:
|
|
|
146
174
|
Tags/ExampleSyntax:
|
|
147
175
|
Description: Validates Ruby syntax in @example tags.
|
|
148
176
|
Enabled: true
|
|
149
|
-
Severity:
|
|
177
|
+
Severity: error
|
|
150
178
|
|
|
151
179
|
Tags/RedundantParamDescription:
|
|
152
180
|
Description: Detects meaningless parameter descriptions that add no value.
|
data/CHANGELOG.md
CHANGED
|
@@ -1,4 +1,68 @@
|
|
|
1
|
-
## [
|
|
1
|
+
## [7.0.4] - 2026-09-23
|
|
2
|
+
|
|
3
|
+
- Fix: `shoryuken sqs dump`/`mv` no longer stop early on real SQS (mensfeld)
|
|
4
|
+
- `find_all` used short polling and broke on the first empty `receive_message` response. Real
|
|
5
|
+
(distributed) SQS routinely returns an empty batch while the queue still has messages, so dump/mv
|
|
6
|
+
quietly processed only a fraction of the queue (invisible on single-node ElasticMQ/LocalStack)
|
|
7
|
+
- It now long-polls and only stops after several consecutive empty batches, so the queue is actually drained
|
|
8
|
+
- Because a longer drain can outlast the queue's visibility timeout (dump/mv delete only after `find_all`
|
|
9
|
+
returns), re-read messages are now deduplicated by message id, so they are not dumped/moved or counted
|
|
10
|
+
twice, and the newest receipt handle is kept so the deferred delete still succeeds
|
|
11
|
+
- Deletion stays deferred (a failed dump/mv deletes nothing), so a queue larger than SQS's in-flight limit
|
|
12
|
+
(~120k standard / ~20k FIFO) drains over multiple runs rather than one - no message is lost
|
|
13
|
+
|
|
14
|
+
- Fix: Graceful stop no longer deadlocks when a manager's dispatch loop never started (mensfeld)
|
|
15
|
+
- `await_dispatching_in_progress` blocks on a `Queue` that is only closed from inside `dispatch_loop` when it
|
|
16
|
+
observes the stop flag. If the loop never runs - a graceful stop arriving before the start `Future` is
|
|
17
|
+
scheduled, or an embedded host whose executor pool is saturated so `manager.start` never runs - the signal
|
|
18
|
+
was never closed and `Launcher#stop` deadlocked
|
|
19
|
+
- `stop_new_dispatching` now closes the release signal itself when the loop hasn't started (tracked via a new
|
|
20
|
+
flag); `Queue#close` is idempotent, so a loop that does start later and closes it again is harmless
|
|
21
|
+
|
|
22
|
+
- Feature: `Shoryuken.fifo_message_deduplication` to opt out of content-based dedup id generation for raw sends (mensfeld)
|
|
23
|
+
- `Queue#add_fifo_attributes!` always set `message_deduplication_id` to a SHA256 of the body when none was
|
|
24
|
+
given, so two raw sends of an identical body within SQS's 5-minute window (`Worker.perform_async`,
|
|
25
|
+
`Queue#send_message`/`#send_messages`) were silently deduplicated - the second dropped. ActiveJob got an
|
|
26
|
+
opt-out in #1017 but the raw path had none
|
|
27
|
+
- Set `Shoryuken.fifo_message_deduplication = false` to stop auto-generating that id, so identical bodies are
|
|
28
|
+
no longer silently dropped (provide a `message_deduplication_id` yourself or enable
|
|
29
|
+
`ContentBasedDeduplication` on the queue)
|
|
30
|
+
- Defaults to `true`, preserving the existing behavior; an explicit `message_deduplication_id` is still honored
|
|
31
|
+
|
|
32
|
+
- Fix: Harden `ExponentialBackoffRetry` visibility handling (mensfeld)
|
|
33
|
+
- `next_visibility_timeout` could return a negative value when a job ran past the 12h SQS ceiling (its
|
|
34
|
+
`max_timeout` goes below zero), and `change_message_visibility` rejects a negative timeout; it now clamps
|
|
35
|
+
to 0 so the message is retried as soon as possible
|
|
36
|
+
- `handle_failure` called `change_visibility` unguarded, so a failure there (e.g. an expired receipt handle)
|
|
37
|
+
escaped the `rescue` in `#call` and masked the original worker error from exception handlers/notifiers; it
|
|
38
|
+
now rescues, logs, and reports "not retried" so the original error is re-raised and the message falls back
|
|
39
|
+
to the queue's default visibility timeout
|
|
40
|
+
- The failure `rescue` in `#call` is now scoped to the single-message `yield` only, so a batch worker's error
|
|
41
|
+
no longer routes the message Array into `handle_failure` (raising a `NoMethodError` that masked the original)
|
|
42
|
+
|
|
43
|
+
- Fix: `perform_async` no longer mutates the caller-supplied options hash (mensfeld)
|
|
44
|
+
- `DefaultExecutor#perform_async` and `InlineExecutor#perform_async` deleted `:queue`, injected
|
|
45
|
+
`:message_body`, and wrote `shoryuken_class` into the nested `:message_attributes` in place, so a caller
|
|
46
|
+
reusing one options hash across enqueues had `:queue` stripped after the first call - silently routing later
|
|
47
|
+
jobs to the worker's default queue
|
|
48
|
+
- Both now operate on a `dup` and rebuild `:message_attributes` with `merge`, leaving the caller's hash untouched
|
|
49
|
+
|
|
50
|
+
- Fix: `Shoryuken::Client.queues` no longer builds the same queue more than once under concurrency (mensfeld)
|
|
51
|
+
- The cache used an unsynchronized `@@queues[name] ||= Shoryuken::Queue.new(...)`. Building a queue makes
|
|
52
|
+
SQS API calls, and that I/O releases the GVL, so concurrent first-access (dispatch, processor-completion
|
|
53
|
+
and worker threads all call it) built the queue multiple times - redundant API calls, and a corrupt cache
|
|
54
|
+
on JRuby/TruffleRuby
|
|
55
|
+
- Access to the cache is now guarded by a mutex
|
|
56
|
+
|
|
57
|
+
- Fix: Busy-processor counter no longer leaks when the executor rejects a worker post (mensfeld)
|
|
58
|
+
- `Manager#assign` increments `@busy_processors` before posting the worker `Concurrent::Promise`, but the
|
|
59
|
+
matching decrement (`processor_done`) runs inside the promise body. When the post is rejected with
|
|
60
|
+
`Concurrent::RejectedExecutionError` - a hard stop racing the `running?` check, or a saturated bounded custom
|
|
61
|
+
`launcher_executor` - the body never runs and the counter leaks
|
|
62
|
+
- With a bounded executor the leak is permanent: `ready` (`@max_processors - busy`) keeps shrinking until
|
|
63
|
+
dispatch stalls and the group silently stops processing
|
|
64
|
+
- The increment is now rolled back on `RejectedExecutionError` by decrementing directly (the message was never
|
|
65
|
+
processed, so the FIFO `message_processed` callback must not run) (#1029)
|
|
2
66
|
|
|
3
67
|
## [7.0.3] - 2026-07-10
|
|
4
68
|
|
data/Gemfile.lint.lock
CHANGED
|
@@ -31,11 +31,11 @@ GEM
|
|
|
31
31
|
unicode-display_width (3.2.0)
|
|
32
32
|
unicode-emoji (~> 4.1)
|
|
33
33
|
unicode-emoji (4.2.0)
|
|
34
|
-
yard (0.9.
|
|
35
|
-
yard-lint (1.
|
|
34
|
+
yard (0.9.45)
|
|
35
|
+
yard-lint (1.10.2)
|
|
36
36
|
yard (~> 0.9)
|
|
37
37
|
zeitwerk (~> 2.6)
|
|
38
|
-
zeitwerk (2.
|
|
38
|
+
zeitwerk (2.8.2)
|
|
39
39
|
|
|
40
40
|
PLATFORMS
|
|
41
41
|
ruby
|
|
@@ -47,6 +47,7 @@ DEPENDENCIES
|
|
|
47
47
|
|
|
48
48
|
CHECKSUMS
|
|
49
49
|
ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383
|
|
50
|
+
bundler (4.0.17) sha256=214e21431b5665dd2f99df8a5511c6b151d7a72e8015c8b38f8b775b61cbb6c1
|
|
50
51
|
json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505
|
|
51
52
|
language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc
|
|
52
53
|
lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87
|
|
@@ -61,9 +62,9 @@ CHECKSUMS
|
|
|
61
62
|
ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33
|
|
62
63
|
unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42
|
|
63
64
|
unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f
|
|
64
|
-
yard (0.9.
|
|
65
|
-
yard-lint (1.
|
|
66
|
-
zeitwerk (2.
|
|
65
|
+
yard (0.9.45) sha256=52e211493f7cb8a3ebf7e104a25a1e73937a3103092545d34cb88fafebb3dc51
|
|
66
|
+
yard-lint (1.10.2) sha256=3e595709d088a8cc827e706a4146b447eb6a907ee375d06c11077369d00717f6
|
|
67
|
+
zeitwerk (2.8.2) sha256=7212a61311083c604184b1ea2574b9aa05cd14f855a0841c06985cabe9181d12
|
|
67
68
|
|
|
68
69
|
BUNDLED WITH
|
|
69
|
-
4.0.
|
|
70
|
+
4.0.17
|
data/bin/cli/sqs.rb
CHANGED
|
@@ -12,6 +12,14 @@ module Shoryuken
|
|
|
12
12
|
# @see https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
|
|
13
13
|
MAX_BATCH_SIZE = 1024 * 1024
|
|
14
14
|
|
|
15
|
+
# Long-poll wait (seconds) used when draining a queue in find_all, so a
|
|
16
|
+
# short-poll empty batch doesn't end dump/mv prematurely.
|
|
17
|
+
FIND_ALL_WAIT_SECONDS = 1
|
|
18
|
+
|
|
19
|
+
# Number of consecutive empty long-poll batches before find_all concludes
|
|
20
|
+
# the queue is drained.
|
|
21
|
+
FIND_ALL_MAX_EMPTY_BATCHES = 3
|
|
22
|
+
|
|
15
23
|
namespace :sqs
|
|
16
24
|
class_option :endpoint, aliases: '-e', type: :string, default: ENV['SHORYUKEN_SQS_ENDPOINT'], desc: 'Endpoint URL'
|
|
17
25
|
|
|
@@ -170,6 +178,25 @@ module Shoryuken
|
|
|
170
178
|
# @return [Integer] the number of messages received
|
|
171
179
|
def find_all(url, limit, &block)
|
|
172
180
|
count = 0
|
|
181
|
+
empty_batches = 0
|
|
182
|
+
# Callers (dump/mv) collect messages and only delete them after this
|
|
183
|
+
# method returns (so a failed dump/mv deletes nothing), which means
|
|
184
|
+
# nothing is removed from the queue during the drain. Two consequences:
|
|
185
|
+
#
|
|
186
|
+
# 1. A large drain can outlast the queue's visibility timeout, at
|
|
187
|
+
# which point already-received messages become visible again and
|
|
188
|
+
# are handed back on a later receive. `seen` maps message id to
|
|
189
|
+
# the first yielded message so those re-reads are not yielded,
|
|
190
|
+
# counted, or dumped/moved twice - but the newest receipt handle
|
|
191
|
+
# is copied onto that message, because SQS only honors the most
|
|
192
|
+
# recently received handle for deletion (an older one can report
|
|
193
|
+
# success yet leave the message in the queue).
|
|
194
|
+
# 2. Because messages stay in flight until the caller deletes them,
|
|
195
|
+
# a queue larger than SQS's in-flight limit (~120k standard /
|
|
196
|
+
# ~20k FIFO) cannot be drained in a single pass: once the limit
|
|
197
|
+
# is reached receive returns empty and find_all stops. No message
|
|
198
|
+
# is lost - re-run dump/mv to drain the remainder.
|
|
199
|
+
seen = {}
|
|
173
200
|
batch_size = limit > 10 ? 10 : limit
|
|
174
201
|
|
|
175
202
|
loop do
|
|
@@ -179,16 +206,46 @@ module Shoryuken
|
|
|
179
206
|
messages = sqs.receive_message(
|
|
180
207
|
queue_url: url,
|
|
181
208
|
max_number_of_messages: batch_size,
|
|
209
|
+
# Long poll: short polling (the default wait_time_seconds: 0)
|
|
210
|
+
# samples only a subset of SQS hosts and routinely returns an empty
|
|
211
|
+
# batch even when the queue still has messages, which made dump/mv
|
|
212
|
+
# stop early and miss messages on real (distributed) SQS.
|
|
213
|
+
wait_time_seconds: FIND_ALL_WAIT_SECONDS,
|
|
182
214
|
attribute_names: ['All'],
|
|
183
215
|
message_attribute_names: ['All']
|
|
184
216
|
).messages || []
|
|
185
217
|
|
|
186
|
-
messages
|
|
218
|
+
# Split first-time messages from re-reads (same id handed back after
|
|
219
|
+
# the visibility timeout lapsed). A batch of only re-reads counts as
|
|
220
|
+
# empty so the drain still terminates.
|
|
221
|
+
fresh = []
|
|
222
|
+
messages.each do |message|
|
|
223
|
+
if (previous = seen[message.message_id])
|
|
224
|
+
# Keep the newest receipt handle on the already-yielded message
|
|
225
|
+
# so the caller's deferred batch_delete targets a handle SQS
|
|
226
|
+
# still accepts.
|
|
227
|
+
previous.receipt_handle = message.receipt_handle
|
|
228
|
+
else
|
|
229
|
+
seen[message.message_id] = message
|
|
230
|
+
fresh << message
|
|
231
|
+
end
|
|
232
|
+
end
|
|
187
233
|
|
|
188
|
-
|
|
234
|
+
fresh.each(&block)
|
|
235
|
+
|
|
236
|
+
count += fresh.size
|
|
189
237
|
|
|
190
238
|
break if count >= limit
|
|
191
|
-
|
|
239
|
+
|
|
240
|
+
# Even with long polling an occasional empty batch is possible while
|
|
241
|
+
# messages remain, so only give up after several consecutive batches
|
|
242
|
+
# that yielded no new messages.
|
|
243
|
+
if fresh.empty?
|
|
244
|
+
empty_batches += 1
|
|
245
|
+
break if empty_batches >= FIND_ALL_MAX_EMPTY_BATCHES
|
|
246
|
+
else
|
|
247
|
+
empty_batches = 0
|
|
248
|
+
end
|
|
192
249
|
end
|
|
193
250
|
|
|
194
251
|
count
|
|
@@ -2,9 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module Shoryuken
|
|
4
4
|
module ActiveJob
|
|
5
|
-
# Adds an accessor for SQS SendMessage parameters on ActiveJob jobs
|
|
6
|
-
#
|
|
7
|
-
# these parameters when enqueueing jobs; other adapters can ignore them.
|
|
5
|
+
# Adds an accessor for SQS SendMessage parameters on ActiveJob jobs (instances of ActiveJob::Base). Shoryuken ActiveJob
|
|
6
|
+
# queue adapters use these parameters when enqueueing jobs; other adapters can ignore them.
|
|
8
7
|
module SQSSendMessageParametersAccessor
|
|
9
8
|
extend ActiveSupport::Concern
|
|
10
9
|
|
|
@@ -14,14 +13,12 @@ module Shoryuken
|
|
|
14
13
|
end
|
|
15
14
|
end
|
|
16
15
|
|
|
17
|
-
# Initializes SQS SendMessage parameters on instances of ActiveJob::Base
|
|
18
|
-
#
|
|
19
|
-
# as when using ActiveJob::Base.set.
|
|
16
|
+
# Initializes SQS SendMessage parameters on instances of ActiveJob::Base to the empty hash, and populates it whenever
|
|
17
|
+
# `#enqueue` is called, such as when using ActiveJob::Base.set.
|
|
20
18
|
module SQSSendMessageParametersSupport
|
|
21
19
|
# Initializes a new ActiveJob instance with empty SQS parameters
|
|
22
20
|
#
|
|
23
|
-
# Uses argument forwarding (...) to properly pass all arguments including
|
|
24
|
-
# keyword arguments to the base class.
|
|
21
|
+
# Uses argument forwarding (...) to properly pass all arguments including keyword arguments to the base class.
|
|
25
22
|
def initialize(...)
|
|
26
23
|
super(...)
|
|
27
24
|
self.sqs_send_message_parameters = {}
|
|
@@ -13,7 +13,7 @@ module ActiveJob
|
|
|
13
13
|
# Shoryuken adapter for Active Job.
|
|
14
14
|
# To use Shoryuken set the queue_adapter config to +:shoryuken+.
|
|
15
15
|
#
|
|
16
|
-
#
|
|
16
|
+
# Example Rails configuration:
|
|
17
17
|
# Rails.application.config.active_job.queue_adapter = :shoryuken
|
|
18
18
|
|
|
19
19
|
# Determine the appropriate base class based on Rails version
|
|
@@ -63,9 +63,8 @@ module ActiveJob
|
|
|
63
63
|
|
|
64
64
|
# Indicates whether Shoryuken is in the process of shutting down.
|
|
65
65
|
#
|
|
66
|
-
# This method is required for ActiveJob Continuations support (Rails 8.1+).
|
|
67
|
-
#
|
|
68
|
-
# and gracefully interrupt execution to allow for resumption after restart.
|
|
66
|
+
# This method is required for ActiveJob Continuations support (Rails 8.1+). When true, it signals to jobs that they
|
|
67
|
+
# should checkpoint their progress and gracefully interrupt execution to allow for resumption after restart.
|
|
69
68
|
#
|
|
70
69
|
# @return [Boolean] true if Shoryuken is shutting down, false otherwise
|
|
71
70
|
# @see https://github.com/rails/rails/pull/55127 Rails ActiveJob Continuations
|
|
@@ -11,12 +11,10 @@ module Shoryuken
|
|
|
11
11
|
# This ensures that request-scoped context (like current user, tenant, locale)
|
|
12
12
|
# automatically flows from the code that enqueues a job to the job's execution.
|
|
13
13
|
#
|
|
14
|
-
# Based on Sidekiq's approach to persisting current attributes, with one
|
|
15
|
-
#
|
|
16
|
-
#
|
|
17
|
-
#
|
|
18
|
-
# loader resets every registered class after each job itself - see the note
|
|
19
|
-
# on {Loading#perform}.
|
|
14
|
+
# Based on Sidekiq's approach to persisting current attributes, with one deliberate difference in cleanup: Sidekiq only
|
|
15
|
+
# touches the classes carried by a job and leaves the general reset to the Rails executor it runs jobs inside. Shoryuken
|
|
16
|
+
# does not run jobs inside that executor by default, so the loader resets every registered class after each job itself -
|
|
17
|
+
# see the note on {Loading#perform}.
|
|
20
18
|
#
|
|
21
19
|
# @example Setup in initializer
|
|
22
20
|
# require 'shoryuken/active_job/current_attributes'
|
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Shoryuken
|
|
4
|
-
# Parses SQS message bodies according to worker configuration.
|
|
5
|
-
# Supports JSON parsing, text extraction, custom Procs, and
|
|
4
|
+
# Parses SQS message bodies according to worker configuration. Supports JSON parsing, text extraction, custom Procs, and
|
|
6
5
|
# any object that responds to parse or load methods.
|
|
7
6
|
class BodyParser
|
|
8
7
|
class << self
|
data/lib/shoryuken/client.rb
CHANGED
|
@@ -4,16 +4,22 @@ module Shoryuken
|
|
|
4
4
|
# Client class for interacting with SQS queues.
|
|
5
5
|
# Provides a simple interface for accessing and managing queue instances.
|
|
6
6
|
class Client
|
|
7
|
-
#
|
|
7
|
+
# Cached queue instances by name (Hash{String => Shoryuken::Queue}).
|
|
8
8
|
@@queues = {}
|
|
9
9
|
|
|
10
|
+
# Guards the queue cache. queues is called concurrently from the dispatch
|
|
11
|
+
# thread, processor-completion threads and worker threads, and building a
|
|
12
|
+
# Shoryuken::Queue makes SQS API calls, so an unsynchronized `||=` would let
|
|
13
|
+
# several callers build the same queue (and corrupt the hash on JRuby).
|
|
14
|
+
@@queues_mutex = Mutex.new
|
|
15
|
+
|
|
10
16
|
class << self
|
|
11
17
|
# Returns a Queue instance for the given queue name
|
|
12
18
|
#
|
|
13
19
|
# @param name [String, Symbol] the name of the queue
|
|
14
20
|
# @return [Shoryuken::Queue] the queue instance
|
|
15
21
|
def queues(name)
|
|
16
|
-
@@queues[name.to_s] ||= Shoryuken::Queue.new(sqs, name)
|
|
22
|
+
@@queues_mutex.synchronize { @@queues[name.to_s] ||= Shoryuken::Queue.new(sqs, name) }
|
|
17
23
|
end
|
|
18
24
|
|
|
19
25
|
# Returns the current SQS client
|
|
@@ -29,10 +35,13 @@ module Shoryuken
|
|
|
29
35
|
# @return [Aws::SQS::Client] the SQS client
|
|
30
36
|
def sqs=(sqs)
|
|
31
37
|
# Since the @@queues values (Shoryuken::Queue objects) are built referencing @@sqs, if it changes, we need to
|
|
32
|
-
# re-build them on subsequent calls to `.queues(name)`.
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
38
|
+
# re-build them on subsequent calls to `.queues(name)`. Swap the client and reset the cache inside the same
|
|
39
|
+
# critical section: otherwise a concurrent `queues` call could slip between the reset and the assignment,
|
|
40
|
+
# read the old client, and cache a queue built with it that survives the replacement.
|
|
41
|
+
@@queues_mutex.synchronize do
|
|
42
|
+
Shoryuken.sqs_client = sqs
|
|
43
|
+
@@queues = {}
|
|
44
|
+
end
|
|
36
45
|
end
|
|
37
46
|
end
|
|
38
47
|
end
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Shoryuken
|
|
4
|
-
# Default exception handler that logs errors during message processing.
|
|
5
|
-
#
|
|
6
|
-
# message and backtrace to the configured logger.
|
|
4
|
+
# Default exception handler that logs errors during message processing. Implements a simple error logging strategy that
|
|
5
|
+
# outputs the exception message and backtrace to the configured logger.
|
|
7
6
|
class DefaultExceptionHandler
|
|
8
7
|
extend Util
|
|
9
8
|
|
data/lib/shoryuken/errors.rb
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Shoryuken
|
|
4
|
-
# Namespace for all Shoryuken-specific errors.
|
|
5
|
-
# These provide more meaningful error types than generic Ruby exceptions,
|
|
4
|
+
# Namespace for all Shoryuken-specific errors. These provide more meaningful error types than generic Ruby exceptions,
|
|
6
5
|
# making it easier to rescue and handle specific failure cases.
|
|
7
6
|
module Errors
|
|
8
7
|
# Base class for all Shoryuken errors
|
|
@@ -14,8 +13,7 @@ module Shoryuken
|
|
|
14
13
|
# Raised when a specified SQS queue does not exist or cannot be accessed
|
|
15
14
|
QueueNotFoundError = Class.new(BaseError)
|
|
16
15
|
|
|
17
|
-
# Raised when worker registration fails due to conflicts
|
|
18
|
-
# (e.g., registering multiple workers for a batch queue)
|
|
16
|
+
# Raised when worker registration fails due to conflicts (e.g., registering multiple workers for a batch queue)
|
|
19
17
|
InvalidWorkerRegistrationError = Class.new(BaseError)
|
|
20
18
|
|
|
21
19
|
# Raised when an invalid polling strategy is specified
|
|
@@ -4,13 +4,11 @@ module Shoryuken
|
|
|
4
4
|
module Helpers
|
|
5
5
|
# A thread-safe counter implementation using Ruby's Mutex.
|
|
6
6
|
#
|
|
7
|
-
# This class provides atomic operations for incrementing, decrementing, and reading
|
|
8
|
-
#
|
|
9
|
-
# Concurrent::AtomicFixnum without requiring external dependencies.
|
|
7
|
+
# This class provides atomic operations for incrementing, decrementing, and reading integer values in a thread-safe
|
|
8
|
+
# manner. It serves as a drop-in replacement for Concurrent::AtomicFixnum without requiring external dependencies.
|
|
10
9
|
#
|
|
11
|
-
# The implementation uses a Mutex to ensure thread safety across all Ruby
|
|
12
|
-
#
|
|
13
|
-
# critical for data integrity.
|
|
10
|
+
# The implementation uses a Mutex to ensure thread safety across all Ruby implementations including JRuby, where true
|
|
11
|
+
# parallelism makes atomic operations critical for data integrity.
|
|
14
12
|
#
|
|
15
13
|
# @note This class is optimized for scenarios with frequent atomic updates
|
|
16
14
|
# and occasional reads, such as tracking active worker counts.
|
|
@@ -82,9 +80,8 @@ module Shoryuken
|
|
|
82
80
|
|
|
83
81
|
# Atomically decrements the counter by 1 and returns the new value.
|
|
84
82
|
#
|
|
85
|
-
# This operation is thread-safe and can be called concurrently from
|
|
86
|
-
#
|
|
87
|
-
# The counter can go negative if decremented below zero.
|
|
83
|
+
# This operation is thread-safe and can be called concurrently from multiple threads without risk of data corruption or
|
|
84
|
+
# lost updates. The counter can go negative if decremented below zero.
|
|
88
85
|
#
|
|
89
86
|
# @return [Integer] The new counter value after decrementing
|
|
90
87
|
#
|
|
@@ -4,14 +4,12 @@ module Shoryuken
|
|
|
4
4
|
module Helpers
|
|
5
5
|
# A thread-safe hash implementation using Ruby's Mutex for all operations.
|
|
6
6
|
#
|
|
7
|
-
# This class provides a hash-like interface with thread-safe operations, serving as a
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
# ensuring complete thread safety across all Ruby implementations including JRuby.
|
|
7
|
+
# This class provides a hash-like interface with thread-safe operations, serving as a drop-in replacement for
|
|
8
|
+
# Concurrent::Hash without requiring external dependencies. The implementation uses a single mutex to protect both read
|
|
9
|
+
# and write operations, ensuring complete thread safety across all Ruby implementations including JRuby.
|
|
11
10
|
#
|
|
12
|
-
# Since hash operations (lookup, assignment) are very fast, the mutex overhead is
|
|
13
|
-
#
|
|
14
|
-
# the complexity of copy-on-write while maintaining excellent performance for
|
|
11
|
+
# Since hash operations (lookup, assignment) are very fast, the mutex overhead is minimal while providing guaranteed
|
|
12
|
+
# safety and simplicity. This approach avoids the complexity of copy-on-write while maintaining excellent performance for
|
|
15
13
|
# typical usage patterns.
|
|
16
14
|
#
|
|
17
15
|
# @note This implementation uses mutex synchronization for all operations,
|
|
@@ -52,8 +50,7 @@ module Shoryuken
|
|
|
52
50
|
class AtomicHash
|
|
53
51
|
# Creates a new empty atomic hash.
|
|
54
52
|
#
|
|
55
|
-
# The hash starts empty and ready to accept key-value pairs through
|
|
56
|
-
# thread-safe operations.
|
|
53
|
+
# The hash starts empty and ready to accept key-value pairs through thread-safe operations.
|
|
57
54
|
#
|
|
58
55
|
# @return [AtomicHash] A new empty atomic hash instance
|
|
59
56
|
#
|
|
@@ -67,8 +64,7 @@ module Shoryuken
|
|
|
67
64
|
|
|
68
65
|
# Returns the value associated with the given key.
|
|
69
66
|
#
|
|
70
|
-
# This operation is thread-safe and will return a consistent value
|
|
71
|
-
# even when called concurrently with write operations.
|
|
67
|
+
# This operation is thread-safe and will return a consistent value even when called concurrently with write operations.
|
|
72
68
|
#
|
|
73
69
|
# @param key [Object] The key to look up
|
|
74
70
|
# @return [Object, nil] The value associated with the key, or nil if not found
|
|
@@ -115,8 +111,7 @@ module Shoryuken
|
|
|
115
111
|
|
|
116
112
|
# Removes all key-value pairs from the hash.
|
|
117
113
|
#
|
|
118
|
-
# This is a thread-safe write operation that ensures atomicity
|
|
119
|
-
# when called concurrently with other operations.
|
|
114
|
+
# This is a thread-safe write operation that ensures atomicity when called concurrently with other operations.
|
|
120
115
|
#
|
|
121
116
|
# @return [Hash] An empty hash (for compatibility with standard Hash#clear)
|
|
122
117
|
#
|
|
@@ -154,8 +149,7 @@ module Shoryuken
|
|
|
154
149
|
|
|
155
150
|
# Returns the value for the given key, or a default value if the key is not found.
|
|
156
151
|
#
|
|
157
|
-
# This operation is thread-safe and will return a consistent value
|
|
158
|
-
# even when called concurrently with write operations.
|
|
152
|
+
# This operation is thread-safe and will return a consistent value even when called concurrently with write operations.
|
|
159
153
|
#
|
|
160
154
|
# @param key [Object] The key to look up
|
|
161
155
|
# @param default [Object] The value to return if the key is not found
|
|
@@ -4,10 +4,9 @@ module Shoryuken
|
|
|
4
4
|
module Helpers
|
|
5
5
|
# Utility methods for hash manipulation.
|
|
6
6
|
#
|
|
7
|
-
# This module provides helper methods for common hash operations that were
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
# the same functionality.
|
|
7
|
+
# This module provides helper methods for common hash operations that were previously implemented as core class
|
|
8
|
+
# extensions. By using a dedicated helper module, we avoid polluting the global namespace while maintaining the same
|
|
9
|
+
# functionality.
|
|
11
10
|
#
|
|
12
11
|
# @example Basic usage
|
|
13
12
|
# hash = { 'key1' => 'value1', 'key2' => { 'nested' => 'value2' } }
|
|
@@ -17,9 +16,8 @@ module Shoryuken
|
|
|
17
16
|
class << self
|
|
18
17
|
# Recursively converts hash keys to symbols.
|
|
19
18
|
#
|
|
20
|
-
# This method traverses a hash structure and converts all string keys
|
|
21
|
-
#
|
|
22
|
-
# This is useful for normalizing configuration data loaded from YAML files.
|
|
19
|
+
# This method traverses a hash structure and converts all string keys to symbols, including nested hashes. Non-hash
|
|
20
|
+
# values are left unchanged. This is useful for normalizing configuration data loaded from YAML files.
|
|
23
21
|
#
|
|
24
22
|
# @param hash [Hash, Object] The hash to convert, or any other object
|
|
25
23
|
# @return [Hash, Object] Hash with symbolized keys, or the original object if not a hash
|
|
@@ -4,10 +4,9 @@ module Shoryuken
|
|
|
4
4
|
module Helpers
|
|
5
5
|
# Utility methods for string manipulation.
|
|
6
6
|
#
|
|
7
|
-
# This module provides helper methods for common string operations that were
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
# the same functionality.
|
|
7
|
+
# This module provides helper methods for common string operations that were previously implemented as core class
|
|
8
|
+
# extensions. By using a dedicated helper module, we avoid polluting the global namespace while maintaining the same
|
|
9
|
+
# functionality.
|
|
11
10
|
#
|
|
12
11
|
# @example Basic usage
|
|
13
12
|
# klass = Shoryuken::Helpers::StringUtils.constantize('MyWorker')
|
|
@@ -16,10 +15,9 @@ module Shoryuken
|
|
|
16
15
|
class << self
|
|
17
16
|
# Converts a string to a constant.
|
|
18
17
|
#
|
|
19
|
-
# This method takes a string representation of a constant name and returns
|
|
20
|
-
#
|
|
21
|
-
#
|
|
22
|
-
# dynamically loading worker classes from configuration.
|
|
18
|
+
# This method takes a string representation of a constant name and returns the actual constant. It handles nested
|
|
19
|
+
# constants (e.g., 'Foo::Bar') and leading double colons (e.g., '::Object'). This is commonly used for dynamically
|
|
20
|
+
# loading worker classes from configuration.
|
|
23
21
|
#
|
|
24
22
|
# @param string [String] The string to convert to a constant
|
|
25
23
|
# @return [Class, Module] The constant represented by the string
|
|
@@ -3,10 +3,9 @@
|
|
|
3
3
|
module Shoryuken
|
|
4
4
|
# A high-performance alternative to OpenStruct for representing SQS messages.
|
|
5
5
|
#
|
|
6
|
-
# InlineMessage is a Struct-based implementation that provides the same interface
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
# needed to represent an Amazon SQS message within the Shoryuken framework.
|
|
6
|
+
# InlineMessage is a Struct-based implementation that provides the same interface as the previous OpenStruct-based message
|
|
7
|
+
# representation but with significantly better performance characteristics. It contains all the essential attributes needed
|
|
8
|
+
# to represent an Amazon SQS message within the Shoryuken framework.
|
|
10
9
|
InlineMessage = Struct.new(
|
|
11
10
|
:body,
|
|
12
11
|
:attributes,
|