shoryuken 7.0.1 → 7.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/push.yml +3 -3
  3. data/.github/workflows/specs.yml +37 -10
  4. data/.github/workflows/verify-action-pins.yml +1 -1
  5. data/.gitignore +0 -1
  6. data/.ruby-version +1 -1
  7. data/CHANGELOG.md +120 -0
  8. data/README.md +1 -1
  9. data/bin/{clean_localstack → clean_sqs} +4 -4
  10. data/bin/cli/sqs.rb +7 -0
  11. data/bin/integrations +52 -34
  12. data/docker-compose.yml +4 -20
  13. data/lib/active_job/queue_adapters/shoryuken_adapter.rb +3 -1
  14. data/lib/shoryuken/active_job/current_attributes.rb +35 -7
  15. data/lib/shoryuken/environment_loader.rb +1 -1
  16. data/lib/shoryuken/fetcher.rb +7 -1
  17. data/lib/shoryuken/helpers/timer_task.rb +19 -2
  18. data/lib/shoryuken/launcher.rb +21 -5
  19. data/lib/shoryuken/manager.rb +38 -5
  20. data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +32 -2
  21. data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +8 -3
  22. data/lib/shoryuken/middleware/server/non_retryable_exception.rb +17 -8
  23. data/lib/shoryuken/options.rb +22 -3
  24. data/lib/shoryuken/polling/strict_priority.rb +26 -14
  25. data/lib/shoryuken/polling/weighted_round_robin.rb +41 -27
  26. data/lib/shoryuken/queue.rb +8 -1
  27. data/lib/shoryuken/util.rb +4 -1
  28. data/lib/shoryuken/version.rb +1 -1
  29. data/lib/shoryuken/worker.rb +5 -1
  30. data/lib/shoryuken.rb +2 -0
  31. data/renovate.json +26 -2
  32. data/spec/integration/active_job/bulk_enqueue/bulk_enqueue_spec.rb +1 -1
  33. data/spec/integration/active_job/current_attributes/bulk_enqueue_spec.rb +1 -1
  34. data/spec/integration/active_job/current_attributes/complex_types_spec.rb +1 -1
  35. data/spec/integration/active_job/current_attributes/cross_job_reset_spec.rb +47 -0
  36. data/spec/integration/active_job/current_attributes/empty_context_spec.rb +1 -1
  37. data/spec/integration/active_job/current_attributes/full_context_spec.rb +1 -1
  38. data/spec/integration/active_job/current_attributes/incremental_persist_spec.rb +76 -0
  39. data/spec/integration/active_job/current_attributes/partial_context_spec.rb +1 -1
  40. data/spec/integration/active_job/custom_attributes/number_attributes_spec.rb +1 -1
  41. data/spec/integration/active_job/custom_attributes/string_attributes_spec.rb +1 -1
  42. data/spec/integration/active_job/fifo_dedup_opt_out/fifo_dedup_opt_out_spec.rb +67 -0
  43. data/spec/integration/active_job/keyword_arguments/keyword_arguments_spec.rb +1 -1
  44. data/spec/integration/active_job/retry/discard_on_spec.rb +1 -1
  45. data/spec/integration/active_job/retry/retry_on_spec.rb +1 -1
  46. data/spec/integration/active_job/roundtrip/roundtrip_spec.rb +2 -2
  47. data/spec/integration/active_job/scheduled/scheduled_spec.rb +1 -1
  48. data/spec/integration/active_record_middleware/active_record_middleware_spec.rb +1 -1
  49. data/spec/integration/auto_delete/auto_delete_spec.rb +1 -1
  50. data/spec/integration/auto_extend_visibility/auto_extend_visibility_spec.rb +1 -1
  51. data/spec/integration/auto_extend_visibility/short_visibility_timeout_spec.rb +52 -0
  52. data/spec/integration/aws_config/aws_config_spec.rb +3 -3
  53. data/spec/integration/batch_processing/batch_processing_spec.rb +1 -1
  54. data/spec/integration/body_parser/json_parser_spec.rb +1 -1
  55. data/spec/integration/body_parser/proc_parser_spec.rb +1 -1
  56. data/spec/integration/body_parser/text_parser_spec.rb +1 -1
  57. data/spec/integration/concurrent_processing/concurrent_processing_spec.rb +1 -1
  58. data/spec/integration/concurrent_processing/processor_accounting_spec.rb +94 -0
  59. data/spec/integration/custom_group_polling_strategy/custom_group_polling_strategy_spec.rb +87 -0
  60. data/spec/integration/dead_letter_queue/dead_letter_queue_spec.rb +1 -1
  61. data/spec/integration/exception_handlers/exception_handlers_spec.rb +1 -1
  62. data/spec/integration/exponential_backoff/exponential_backoff_spec.rb +1 -1
  63. data/spec/integration/fifo_ordering/fifo_max_messages_cap_spec.rb +96 -0
  64. data/spec/integration/fifo_ordering/fifo_ordering_spec.rb +1 -1
  65. data/spec/integration/large_payloads/large_payloads_spec.rb +1 -1
  66. data/spec/integration/launcher/double_graceful_stop_spec.rb +71 -0
  67. data/spec/integration/launcher/embedded_dispatch_error_spec.rb +85 -0
  68. data/spec/integration/launcher/global_executor_preserved_spec.rb +76 -0
  69. data/spec/integration/launcher/graceful_stop_timeout_spec.rb +74 -0
  70. data/spec/integration/launcher/launcher_spec.rb +1 -1
  71. data/spec/integration/message_attributes/message_attributes_spec.rb +1 -1
  72. data/spec/integration/message_operations/message_operations_spec.rb +1 -1
  73. data/spec/integration/message_operations/partial_batch_delete_spec.rb +67 -0
  74. data/spec/integration/non_retryable_exception/non_retryable_exception_spec.rb +2 -2
  75. data/spec/integration/non_retryable_exception/with_retry_intervals_spec.rb +115 -0
  76. data/spec/integration/polling_strategies/polling_strategies_spec.rb +1 -1
  77. data/spec/integration/queue_operations/queue_operations_spec.rb +1 -1
  78. data/spec/integration/retry_behavior/retry_behavior_spec.rb +1 -1
  79. data/spec/integration/strict_priority_polling/strict_priority_polling_spec.rb +1 -1
  80. data/spec/integration/visibility_timeout/visibility_timeout_spec.rb +1 -1
  81. data/spec/integration/worker_enqueueing/worker_enqueueing_spec.rb +1 -1
  82. data/spec/integration/worker_groups/worker_groups_spec.rb +1 -1
  83. data/spec/integration/worker_lifecycle/worker_lifecycle_spec.rb +1 -1
  84. data/spec/integrations_helper.rb +13 -12
  85. data/spec/lib/shoryuken/fetcher_spec.rb +13 -0
  86. data/spec/lib/shoryuken/helpers/timer_task_spec.rb +24 -0
  87. data/spec/lib/shoryuken/launcher_spec.rb +38 -0
  88. data/spec/lib/shoryuken/manager_spec.rb +99 -0
  89. data/spec/lib/shoryuken/middleware/server/auto_extend_visibility_spec.rb +35 -0
  90. data/spec/lib/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +56 -0
  91. data/spec/lib/shoryuken/options_spec.rb +43 -0
  92. data/spec/lib/shoryuken/polling/strict_priority_spec.rb +25 -0
  93. data/spec/lib/shoryuken/polling/weighted_round_robin_spec.rb +50 -0
  94. data/spec/lib/shoryuken/queue_spec.rb +37 -0
  95. data/spec/lib/shoryuken/util_spec.rb +26 -0
  96. data/spec/shared_examples_for_active_job.rb +18 -0
  97. data/spec/spec_helper.rb +26 -7
  98. metadata +29 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 40068ac6b1cebd740c3e9180956731b624a541f967987d0f5f18505ab3885916
4
- data.tar.gz: dd58ae5cc228e509665010320bc833c6e8c3377961b7128153406953e3d0c9ed
3
+ metadata.gz: 988d29701fd4b73dea7d965b283ae8d3fccf5b9737a7e3acdfe5cc760ca82cc5
4
+ data.tar.gz: b91a7b40cab94ca17fb432433a81323deba9675a3398678c80db1b3a7aa454fb
5
5
  SHA512:
6
- metadata.gz: 645962569cc872e4709f0b8dcbeac166b7f398459144f088cfab13addcf86f85d494c604591623760c64a126b002b1dcaac64ffda68003ad3c783beff3b02b8a
7
- data.tar.gz: adf758d00496d6088d49ed0c456915bcdb18359c9ba80560d97cda1daca526b1233d266a65989851f23cd18887636399aa5836071d12b7b45cb406b241ed3014
6
+ metadata.gz: e6bc156e3e5387ad74c4a740cae775f6f633b67c26395a3aac11e51b7cc47771785918e5ec457d3f2317148846ef3ad0b137b5f6e012f3f75aec6536a2eb2deb
7
+ data.tar.gz: 87f0feeae55309458dbd5eab9fea4fbdb83a5689bd526b0e22857cb89f0af7027346f2e1487614965b3208aafaf6167dc7bb2573e160d45cfb54e8133baa09c0
@@ -19,12 +19,12 @@ jobs:
19
19
  id-token: write
20
20
 
21
21
  steps:
22
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
22
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
23
23
  with:
24
24
  fetch-depth: 0
25
25
 
26
26
  - name: Set up Ruby
27
- uses: ruby/setup-ruby@8d27f39a5e7ad39aebbcbd1324f7af020229645c # v1.287.0
27
+ uses: ruby/setup-ruby@d45b1a4e94b71acab930e56e79c6aa188764e7f9 # v1.316.0
28
28
  with:
29
29
  bundler-cache: false
30
30
 
@@ -33,4 +33,4 @@ jobs:
33
33
  bundle install --jobs 4 --retry 3
34
34
 
35
35
  # Release
36
- - uses: rubygems/release-gem@1c162a739e8b4cb21a676e97b087e8268d8fc40b # v1.1.2
36
+ - uses: rubygems/release-gem@052cc82692552de3ef2b81fd670e41d13cba8092 # v1.4.0
@@ -11,17 +11,41 @@ jobs:
11
11
  matrix:
12
12
  ruby: ['3.2', '3.3', '3.4', '4.0']
13
13
  runs-on: ubuntu-latest
14
+ # Unit specs finish in under a minute; fail fast instead of letting a
15
+ # wedged runner occupy the job slot for the 6-hour default
16
+ timeout-minutes: 10
14
17
  steps:
15
18
  - name: Checkout code
16
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
19
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
17
20
 
18
- - uses: ruby/setup-ruby@8d27f39a5e7ad39aebbcbd1324f7af020229645c # v1.287.0
21
+ - uses: ruby/setup-ruby@d45b1a4e94b71acab930e56e79c6aa188764e7f9 # v1.316.0
19
22
  with:
20
23
  ruby-version: ${{ matrix.ruby }}
21
24
  bundler-cache: true
22
25
 
23
26
  - name: Run specs
24
- run: bundle exec rake spec
27
+ run: |
28
+ # Run rspec directly (not via rake) so $SPEC_PID is the ruby process
29
+ # itself — the USR1 watchdog below sends the signal to that pid, and
30
+ # spec_helper registers a USR1 handler that prints a thread dump.
31
+ # Running via 'bundle exec rake spec' would send the signal to rake,
32
+ # which has no USR1 handler and would crash before the dump appears.
33
+ bundle exec rspec --format documentation &
34
+ SPEC_PID=$!
35
+ # Send USR1 after 8 minutes so the thread-dump handler in spec_helper
36
+ # prints a backtrace before the 10-minute job timeout kills everything.
37
+ ( sleep 480
38
+ if kill -0 $SPEC_PID 2>/dev/null; then
39
+ echo "=== specs still running after 8 minutes — requesting thread dump ==="
40
+ kill -USR1 $SPEC_PID 2>/dev/null || true
41
+ sleep 5
42
+ fi
43
+ ) &
44
+ WATCHDOG_PID=$!
45
+ wait $SPEC_PID
46
+ EXIT_CODE=$?
47
+ kill $WATCHDOG_PID 2>/dev/null || true
48
+ exit $EXIT_CODE
25
49
 
26
50
  integrations:
27
51
  name: Integrations
@@ -29,23 +53,25 @@ jobs:
29
53
  matrix:
30
54
  ruby: ['3.2', '3.3', '3.4', '4.0']
31
55
  runs-on: ubuntu-latest
56
+ # Integration specs normally take ~4 minutes
57
+ timeout-minutes: 15
32
58
  steps:
33
59
  - name: Checkout code
34
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
60
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
35
61
 
36
- - name: Start LocalStack
62
+ - name: Start ElasticMQ
37
63
  run: docker compose up -d
38
64
 
39
- - name: Wait for LocalStack
65
+ - name: Wait for ElasticMQ
40
66
  run: |
41
67
  timeout 30s bash -c '
42
- until curl -s http://localhost:4566/_localstack/health | grep -q "\"sqs\": \"available\""; do
43
- echo "Waiting for LocalStack..."
44
- sleep 2
68
+ until curl -s http://localhost:9324/health; do
69
+ echo "Waiting for ElasticMQ..."
70
+ sleep 1
45
71
  done
46
72
  '
47
73
 
48
- - uses: ruby/setup-ruby@8d27f39a5e7ad39aebbcbd1324f7af020229645c # v1.287.0
74
+ - uses: ruby/setup-ruby@d45b1a4e94b71acab930e56e79c6aa188764e7f9 # v1.316.0
49
75
  with:
50
76
  ruby-version: ${{ matrix.ruby }}
51
77
  bundler-cache: true
@@ -56,6 +82,7 @@ jobs:
56
82
  ci-success:
57
83
  name: CI Success
58
84
  runs-on: ubuntu-latest
85
+ timeout-minutes: 5
59
86
  if: always()
60
87
  needs:
61
88
  - specs
@@ -7,7 +7,7 @@ jobs:
7
7
  verify:
8
8
  runs-on: ubuntu-latest
9
9
  steps:
10
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
10
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
11
11
  - name: Check SHA pins
12
12
  run: |
13
13
  if grep -E -r "uses: .*/.*@(v[0-9]+|main|master)($|[[:space:]]|$)" --include="*.yml" --include="*.yaml" .github/workflows/ | grep -v "#"; then
data/.gitignore CHANGED
@@ -26,7 +26,6 @@ shoryuken.yml
26
26
  .env
27
27
  rubocop.html
28
28
  .byebug_history
29
- .localstack
30
29
  spec/integration/**/Gemfile.lock
31
30
  spec/integration/**/vendor/
32
31
  spec/integration/**/.bundle/
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 4.0.1
1
+ 4.0.5
data/CHANGELOG.md CHANGED
@@ -1,3 +1,123 @@
1
+ ## [Unreleased]
2
+
3
+ ## [7.0.3] - 2026-07-10
4
+
5
+ - Feature: `Shoryuken.active_job_fifo_message_deduplication` to opt out of FIFO dedup id generation (mensfeld)
6
+ - For FIFO queues the ActiveJob adapter derives a content-based `message_deduplication_id` from the
7
+ serialized job minus `job_id`/`enqueued_at` (#457 / #750), so two distinct enqueues of the same job
8
+ class and arguments within SQS's 5-minute window silently collapse into one - a "skipped message" trap
9
+ - Set `Shoryuken.active_job_fifo_message_deduplication = false` to stop generating that id, so identical
10
+ jobs are no longer silently dropped (rely on the queue's content-based deduplication or explicit ids)
11
+ - Defaults to `true`, preserving the existing behavior; an explicit `message_deduplication_id` is still honored
12
+
13
+ - Fix: Polling strategies are now thread-safe, and WeightedRoundRobin unpauses processed queues reliably (mensfeld)
14
+ - `message_processed` runs on processor-completion threads (for FIFO queues) while `next_queue`/`messages_found`
15
+ run on the dispatch thread; they mutate the same state with no synchronization, which is benign on MRI (GVL)
16
+ but corrupts state on JRuby/TruffleRuby. Both `WeightedRoundRobin` and `StrictPriority` now serialize access
17
+ with a mutex
18
+ - `WeightedRoundRobin#unpause_queues` only checked the head of the paused list, so a queue marked ready by
19
+ `message_processed` could stay stuck behind an earlier-paused queue; it now unpauses the first expired entry
20
+ anywhere in the list
21
+
22
+ - Fix: `TimerTask#kill` no longer deadlocks on Ruby 3.2 under concurrent callers (mensfeld)
23
+ - `kill` called `@thread.kill` while holding `@mutex`; the timer loop's `ensure` block calls
24
+ `@mutex.synchronize` to clear `@running`, so on Ruby 3.2 (where `Thread#kill` yields the GVL
25
+ to the killed thread for cleanup before returning) both threads waited on each other forever
26
+ - The thread is now killed after the mutex is released, so the ensure block can always acquire it
27
+
28
+ - Fix: Stopping the launcher no longer destroys the process-global IO executor (mensfeld)
29
+ - With no `launcher_executor` configured, `Launcher#executor` fell back to `Concurrent.global_io_executor`,
30
+ and `Launcher#stop`/`#stop!` call `shutdown` (and `kill`) on it
31
+ - Shutting down that process-wide pool broke anything else relying on concurrent-ruby's `:io` pool
32
+ (including Shoryuken's own `ShoryukenConcurrentSendAdapter`) and prevented starting a fresh launcher
33
+ in the same process
34
+ - The launcher now owns a dedicated `Concurrent::CachedThreadPool`, so stopping it leaves the global
35
+ IO executor untouched
36
+
37
+ - Fix: `Queue#delete_messages` now logs every batch-delete failure and returns a robust boolean (mensfeld)
38
+ - It used `failed.any? { |f| logger.error ... }`, which short-circuited after the first failure - so when
39
+ a batch had multiple failures only the first was logged - and only returned truthy because `Logger#error`
40
+ happens to return true (a custom logger returning falsey would have hidden the failure from callers)
41
+ - It now logs each failure and returns `failed.any?`, so `NonRetryableException`/`AutoDelete` reliably see
42
+ that some messages may remain and need reprocessing
43
+
44
+ - Fix: Lifecycle events fired in reverse no longer alternate handler order (mensfeld)
45
+ - `Util#fire_event` reversed the stored handler array in place with `reverse!`, so an event fired more
46
+ than once with `reverse: true` (e.g. `:shutdown` via `stop` then `stop!`) flipped order each time
47
+ - It now reverses a copy, leaving the stored handler order untouched
48
+
49
+ - Fix: A fatal dispatch error no longer hard-kills an embedded host process (mensfeld)
50
+ - `Manager#handle_dispatch_error` sent `Process.kill('USR1', Process.pid)` unconditionally after a
51
+ dispatch error (e.g. SQS still failing once the fetcher exhausted its retries)
52
+ - The CLI Runner traps USR1 and turns it into a graceful shutdown, but a host embedding
53
+ `Shoryuken::Launcher` directly has USR1's default disposition, so the whole process - and its
54
+ in-flight workers - was terminated
55
+ - When embedded (no CLI Runner), the failing manager now just stops itself and `Launcher#healthy?`
56
+ reports the failure; the USR1 signal is only sent in server (CLI) mode, preserving the
57
+ supervisor-restart behavior there
58
+
59
+ - Fix: Graceful stop is now bounded by the configured timeout (mensfeld)
60
+ - `Launcher#stop` (the soft shutdown behind USR1/TSTP) called `executor.wait_for_termination` with no
61
+ argument - an unbounded wait - so a single hung worker blocked shutdown forever
62
+ - Both `Launcher#stop` and `Launcher#stop!` now share a `shutdown_executor` helper that waits up to
63
+ `Shoryuken.options[:timeout]` seconds for in-flight workers, then force-kills the executor so the
64
+ process can exit; the graceful stop still waits for workers, just no longer indefinitely
65
+
66
+ - Docs: Correct the `retry_intervals` exponential backoff documentation (mensfeld)
67
+ - The `exponential_backoff?` docstring claimed retries stop ("before giving up") after the last configured
68
+ interval. They do not: once the intervals are exhausted, the last interval is reused for every later
69
+ attempt, and SQS's redrive policy (maxReceiveCount) is what ultimately moves a message to a dead-letter queue
70
+ - Added a spec pinning that far-later attempts keep reusing the last interval
71
+
72
+ - Fix: `CurrentAttributes.persist` no longer drops a class when called once per class (mensfeld)
73
+ - The storage key was derived from the per-call index, so registering classes across separate `persist`
74
+ calls made the third call reuse `cattr_0` and silently overwrite the second class - its attributes were
75
+ then never serialized or restored
76
+ - The key now uses the running registry size, so incremental and single-call registration both yield
77
+ distinct, stable keys (single-call `persist(A, B, C)` keys are unchanged)
78
+
79
+ - Fix: Busy-processor accounting no longer breaks when processor completion raises (mensfeld)
80
+ - `Manager#assign` chained `.then { processor_done }.rescue { processor_done }`, so an exception inside
81
+ `processor_done` (SQS lookups or a polling strategy's `message_processed` callback) ran completion twice
82
+ - The busy counter was decremented twice for one message and drifted negative, inflating `ready` and
83
+ silently breaking the configured concurrency limit for the life of the process
84
+ - Completion now runs in an `ensure` around processing (exactly once), and `processor_done` logs
85
+ instead of leaking exceptions from the FIFO bookkeeping
86
+
87
+ - Fix: Repeated graceful stop no longer deadlocks the process (mensfeld)
88
+ - `Manager#await_dispatching_in_progress` popped a signal queue that received exactly one token,
89
+ so a second `Launcher#stop` blocked forever on an empty queue
90
+ - Operationally this was the TSTP -> USR1 sequence: both signals trigger a graceful stop, leaving
91
+ the process stuck in the signal loop where even TERM/INT were no longer handled (only SIGKILL worked)
92
+ - The dispatch loop now closes the signal queue instead of pushing a token, releasing every pending
93
+ and future waiter; the dispatch chain also releases waiters if the executor rejects a post during
94
+ a racing hard shutdown
95
+
96
+ - Fix: `non_retryable_exceptions` is no longer ignored when `retry_intervals` is also configured (mensfeld)
97
+ - `ExponentialBackoffRetry` swallowed every exception after scheduling a retry, so `NonRetryableException`
98
+ (which sits outside it in the default middleware chain) never saw non-retryable errors - poison messages
99
+ were retried indefinitely (with the last interval repeated) instead of being deleted immediately
100
+ - `ExponentialBackoffRetry` now re-raises exceptions classified as non-retryable so the message gets deleted
101
+ - Exception classification is extracted to `NonRetryableException.non_retryable?` and shared by both middlewares
102
+
103
+ - Enhancement: Use dynamic Ruby warning category opt-in in test helpers (mensfeld)
104
+ - Replace version-gated `Warning[:performance]` with `Warning.categories`-based auto-enablement
105
+ - Automatically enables all non-deprecated, non-experimental warning categories for forward compatibility
106
+ - Applied to both `spec/spec_helper.rb` and `spec/integrations_helper.rb`
107
+
108
+ ## [7.0.2] - 2026-04-16
109
+
110
+ - Enhancement: Replace LocalStack with ElasticMQ for SQS integration tests (mensfeld)
111
+ - ElasticMQ is free, open-source, and requires no auth token (LocalStack 2026.x requires paid account)
112
+ - Uses `softwaremill/elasticmq-native` image (~30MB vs ~1GB+, starts in milliseconds)
113
+ - Rename `setup_localstack` to `setup_sqs` and `bin/clean_localstack` to `bin/clean_sqs`
114
+
115
+ - Fix: Allow custom polling strategy to be configured per-group via `add_group` (mensfeld)
116
+ - `add_group` now accepts `polling_strategy:` keyword argument
117
+ - `polling_strategy()` reads from the groups hash populated by `add_group`, with fallback to raw options
118
+ - `EnvironmentLoader` passes `polling_strategy` through when parsing YAML group config
119
+ - [#925](https://github.com/ruby-shoryuken/shoryuken/issues/925)
120
+
1
121
  ## [7.0.1] - 2026-02-07
2
122
 
3
123
  - Enhancement: Add non-retryable exception middleware (Saidbek)
data/README.md CHANGED
@@ -63,7 +63,7 @@ To run all unit specs against the latest dependency versions, execute
63
63
  bundle exec rake spec
64
64
  ```
65
65
 
66
- To run integration specs (including Rails tests), start LocalStack and run:
66
+ To run integration specs (including Rails tests), start ElasticMQ and run:
67
67
 
68
68
  ```sh
69
69
  docker compose up -d
@@ -1,14 +1,14 @@
1
1
  #!/usr/bin/env ruby
2
2
  # frozen_string_literal: true
3
3
 
4
- # Removes all integration test SQS queues from LocalStack
4
+ # Removes all integration test SQS queues from ElasticMQ
5
5
  #
6
- # Useful when having a long-running LocalStack instance that cannot be fully
6
+ # Useful when having a long-running ElasticMQ instance that cannot be fully
7
7
  # restarted between test runs. All integration test queues use the 'it-' prefix,
8
8
  # making them easy to identify and remove.
9
9
  #
10
10
  # Usage:
11
- # bin/clean_localstack
11
+ # bin/clean_sqs
12
12
 
13
13
  require 'aws-sdk-sqs'
14
14
 
@@ -16,7 +16,7 @@ THREADS_COUNT = 3
16
16
 
17
17
  sqs = Aws::SQS::Client.new(
18
18
  region: 'us-east-1',
19
- endpoint: 'http://localhost:4566',
19
+ endpoint: 'http://localhost:9324',
20
20
  access_key_id: 'fake',
21
21
  secret_access_key: 'fake'
22
22
  )
data/bin/cli/sqs.rb CHANGED
@@ -146,6 +146,13 @@ module Shoryuken
146
146
  value[:string_value].bytesize
147
147
  elsif value[:binary_value]
148
148
  value[:binary_value].bytesize
149
+ else
150
+ # List-type attributes (string_list_values /
151
+ # binary_list_values) carry no scalar value here and
152
+ # are intentionally not requeued (see
153
+ # normalize_dump_message). Size them as 0 rather than
154
+ # crashing the whole batch with `nil + Integer`.
155
+ 0
149
156
  end
150
157
  name_size + data_type_size + value_size
151
158
  end
data/bin/integrations CHANGED
@@ -70,21 +70,23 @@ class IntegrationRunner
70
70
  def run_specs(specs)
71
71
  results = []
72
72
 
73
- specs.each do |spec|
73
+ specs.each_with_index do |spec, idx|
74
+ prefix = "[#{idx + 1}/#{specs.size}]"
75
+ puts "#{prefix} RUNNING #{spec[:name]}"
76
+ $stdout.flush
77
+
74
78
  result = run_spec(spec)
75
79
  results << result
76
80
 
77
- if result[:skipped]
78
- print 'S'
79
- elsif result[:success]
80
- print '.'
81
- else
82
- print 'F'
83
- end
81
+ elapsed = result[:duration] ? format('%.1fs', result[:duration]) : '?'
82
+ status = if result[:skipped] then 'SKIPPED'
83
+ elsif result[:success] then 'OK '
84
+ else 'FAILED '
85
+ end
86
+ puts "#{prefix} #{status} #{spec[:name]} (#{elapsed})"
84
87
  $stdout.flush
85
88
  end
86
89
 
87
- puts
88
90
  results
89
91
  end
90
92
 
@@ -132,34 +134,50 @@ class IntegrationRunner
132
134
  output = []
133
135
  start_time = Time.now
134
136
 
135
- begin
136
- Timeout.timeout(TIMEOUT) do
137
- # Use unbundled env to prevent pollution from previous test runs
138
- # This is especially important after Rails integration tests that use
139
- # bundle install --standalone with different gem versions
140
- Bundler.with_unbundled_env do
141
- IO.popen(env, cmd, chdir: spec[:directory], err: [:child, :out]) do |io|
142
- io.each_line { |line| output << line }
143
- end
137
+ Bundler.with_unbundled_env do
138
+ # Use non-block popen so we control when waitpid runs. The block form
139
+ # calls io.close (which calls waitpid internally) as part of its own
140
+ # ensure, so if the child is still running when Timeout::Error fires,
141
+ # io.close blocks indefinitely before we can rescue the error.
142
+ io = IO.popen(env, cmd, chdir: spec[:directory], err: [:child, :out])
143
+ timed_out = false
144
+
145
+ begin
146
+ Timeout.timeout(TIMEOUT) do
147
+ io.each_line { |line| output << line }
148
+ end
149
+ rescue Timeout::Error
150
+ timed_out = true
151
+ ensure
152
+ # Kill before closing: io.close calls waitpid, which blocks if the
153
+ # child is still alive. Sending SIGKILL first guarantees the child
154
+ # exits promptly so waitpid (inside io.close) returns immediately.
155
+ begin
156
+ Process.kill('KILL', io.pid)
157
+ rescue Errno::ESRCH
158
+ # Child already exited on its own — nothing to kill
144
159
  end
160
+ io.close rescue nil
145
161
  end
146
162
 
147
- {
148
- spec: spec,
149
- success: $?.success?,
150
- exit_code: $?.exitstatus,
151
- duration: Time.now - start_time,
152
- output: output.join
153
- }
154
- rescue Timeout::Error
155
- {
156
- spec: spec,
157
- success: false,
158
- exit_code: -1,
159
- error: 'Timeout',
160
- duration: Time.now - start_time,
161
- output: output.join
162
- }
163
+ if timed_out
164
+ {
165
+ spec: spec,
166
+ success: false,
167
+ exit_code: -1,
168
+ error: 'Timeout',
169
+ duration: Time.now - start_time,
170
+ output: output.join
171
+ }
172
+ else
173
+ {
174
+ spec: spec,
175
+ success: $?.success?,
176
+ exit_code: $?.exitstatus,
177
+ duration: Time.now - start_time,
178
+ output: output.join
179
+ }
180
+ end
163
181
  end
164
182
  end
165
183
 
data/docker-compose.yml CHANGED
@@ -1,22 +1,6 @@
1
1
  services:
2
- localstack:
3
- image: localstack/localstack:latest
4
- container_name: localstack
2
+ elasticmq:
3
+ image: softwaremill/elasticmq-native:1.7.1
4
+ container_name: elasticmq
5
5
  ports:
6
- - "4566:4566"
7
- - "4510-4559:4510-4559"
8
- environment:
9
- - SERVICES=sqs
10
- - DEBUG=${DEBUG:-0}
11
- - DOCKER_HOST=unix:///var/run/docker.sock
12
- - AWS_DEFAULT_REGION=us-east-1
13
- - AWS_ACCESS_KEY_ID=test
14
- - AWS_SECRET_ACCESS_KEY=test
15
- volumes:
16
- - "${LOCALSTACK_VOLUME_DIR:-./.localstack}:/var/lib/localstack"
17
- - "/var/run/docker.sock:/var/run/docker.sock"
18
- healthcheck:
19
- test: ["CMD", "curl", "-f", "http://localhost:4566/_localstack/health"]
20
- interval: 5s
21
- timeout: 10s
22
- retries: 5
6
+ - "9324:9324"
@@ -177,9 +177,11 @@ module ActiveJob
177
177
  message_attributes: attributes.merge(MESSAGE_ATTRIBUTES)
178
178
  }
179
179
 
180
- if queue.fifo?
180
+ if queue.fifo? && Shoryuken.active_job_fifo_message_deduplication?
181
181
  # See https://github.com/ruby-shoryuken/shoryuken/issues/457 and
182
182
  # https://github.com/ruby-shoryuken/shoryuken/pull/750#issuecomment-1781317929
183
+ # Disable via Shoryuken.active_job_fifo_message_deduplication = false when distinct
184
+ # enqueues of the same job class and arguments must not be silently deduplicated.
183
185
  msg[:message_deduplication_id] = Digest::SHA256.hexdigest(
184
186
  JSON.dump(body.except('job_id', 'enqueued_at'))
185
187
  )
@@ -11,7 +11,12 @@ module Shoryuken
11
11
  # This ensures that request-scoped context (like current user, tenant, locale)
12
12
  # automatically flows from the code that enqueues a job to the job's execution.
13
13
  #
14
- # Based on Sidekiq's approach to persisting current attributes.
14
+ # Based on Sidekiq's approach to persisting current attributes, with one
15
+ # deliberate difference in cleanup: Sidekiq only touches the classes carried
16
+ # by a job and leaves the general reset to the Rails executor it runs jobs
17
+ # inside. Shoryuken does not run jobs inside that executor by default, so the
18
+ # loader resets every registered class after each job itself - see the note
19
+ # on {Loading#perform}.
15
20
  #
16
21
  # @example Setup in initializer
17
22
  # require 'shoryuken/active_job/current_attributes'
@@ -58,8 +63,12 @@ module Shoryuken
58
63
  def persist(*klasses)
59
64
  @cattrs ||= {}
60
65
 
61
- klasses.flatten.each_with_index do |klass, idx|
62
- key = @cattrs.empty? ? 'cattr' : "cattr_#{idx}"
66
+ klasses.flatten.each do |klass|
67
+ # Key off the running registry size, not the per-call index, so that
68
+ # registering classes across separate persist calls still produces
69
+ # distinct keys (a per-call index restarts at 0 each call and would
70
+ # overwrite earlier registrations).
71
+ key = @cattrs.empty? ? 'cattr' : "cattr_#{@cattrs.size}"
63
72
  @cattrs[key] = klass.to_s
64
73
  end
65
74
 
@@ -109,13 +118,10 @@ module Shoryuken
109
118
  # @param hash [Hash] the deserialized job data
110
119
  # @return [void]
111
120
  def perform(sqs_msg, hash)
112
- klasses_to_reset = []
113
-
114
121
  CurrentAttributes.cattrs&.each do |key, klass_name|
115
122
  next unless hash.key?(key)
116
123
 
117
124
  klass = klass_name.constantize
118
- klasses_to_reset << klass
119
125
 
120
126
  begin
121
127
  attrs = Serializer.deserialize(hash[key])
@@ -131,7 +137,29 @@ module Shoryuken
131
137
 
132
138
  super
133
139
  ensure
134
- klasses_to_reset.each(&:reset)
140
+ # Reset every registered CurrentAttributes class after the job - not
141
+ # only the ones whose key was in this message.
142
+ #
143
+ # Why unconditional (and why this differs from Sidekiq): Sidekiq's
144
+ # loader only touches classes present in the job and relies on the
145
+ # Rails executor - which it runs every job inside - to reset all
146
+ # CurrentAttributes between units of work. Shoryuken has no such safety
147
+ # net: it wraps a job in the reloader/executor only when
148
+ # `enable_reloading` is set, which is off by default, so nothing else
149
+ # clears CurrentAttributes between jobs.
150
+ #
151
+ # A blanket reset here is therefore the only thing guaranteeing a clean
152
+ # thread. Resetting only the present keys leaks whenever a value ends up
153
+ # set during a job whose message carried no cattr key - e.g. the worker
154
+ # (or code it calls) writes to Current, on a keyless message (empty
155
+ # context at enqueue, a different producer, or persist configured after
156
+ # the message was queued). CurrentAttributes are thread-local and the
157
+ # pool reuses threads, so that value would surface in the next job.
158
+ CurrentAttributes.cattrs&.each_value do |klass_name|
159
+ klass_name.constantize.reset
160
+ rescue => e
161
+ Shoryuken.logger.warn("Failed to reset CurrentAttributes #{klass_name}: #{e.message}")
162
+ end
135
163
  end
136
164
  end
137
165
  end
@@ -200,7 +200,7 @@ module Shoryuken
200
200
  end
201
201
 
202
202
  Shoryuken.options[:groups].to_a.each do |group, options|
203
- Shoryuken.add_group(group, options[:concurrency], delay: options[:delay])
203
+ Shoryuken.add_group(group, options[:concurrency], delay: options[:delay], polling_strategy: options[:polling_strategy])
204
204
 
205
205
  options[:queues].to_a.each do |queue, weight|
206
206
  parse_queue(queue, weight, group)
@@ -72,12 +72,18 @@ module Shoryuken
72
72
 
73
73
  shoryuken_queue = Shoryuken::Client.queues(queue.name)
74
74
 
75
- options[:max_number_of_messages] = max_number_of_messages(shoryuken_queue, limit, options)
76
75
  options[:message_attribute_names] = %w[All]
77
76
  options[:attribute_names] = %w[All]
78
77
 
78
+ # Merge per-queue options BEFORE computing the cap so the FIFO
79
+ # one-at-a-time guard (and FETCH_LIMIT) always win. Computing the cap last
80
+ # means a queue option of max_number_of_messages can only lower the count,
81
+ # never raise a non-batch FIFO queue above 1 - which would let SQS return
82
+ # several messages from the same group and break ordering.
79
83
  options.merge!(queue.options)
80
84
 
85
+ options[:max_number_of_messages] = max_number_of_messages(shoryuken_queue, limit, options)
86
+
81
87
  shoryuken_queue.receive_messages(options)
82
88
  end
83
89
 
@@ -42,14 +42,21 @@ module Shoryuken
42
42
  #
43
43
  # @return [Boolean] true if killed, false if already killed
44
44
  def kill
45
+ thread_to_kill = nil
46
+
45
47
  @mutex.synchronize do
46
48
  return false if @killed
47
49
 
48
50
  @killed = true
49
51
  @running = false
50
-
51
- @thread.kill if @thread&.alive?
52
+ thread_to_kill = @thread
52
53
  end
54
+
55
+ # Kill the thread AFTER releasing the mutex. The timer loop's ensure
56
+ # block calls @mutex.synchronize to clear @running; killing the thread
57
+ # while holding that mutex deadlocks on Ruby 3.2, where Thread#kill
58
+ # yields the GVL to the killed thread for cleanup before returning.
59
+ thread_to_kill&.kill if thread_to_kill&.alive?
53
60
  true
54
61
  end
55
62
 
@@ -59,6 +66,16 @@ module Shoryuken
59
66
  #
60
67
  # @return [void]
61
68
  def run_timer_loop
69
+ # The timer thread inherits the priority of the thread that called
70
+ # #execute. Shoryuken runs workers at a lowered priority
71
+ # (Shoryuken.thread_priority, default -1) and starts the
72
+ # auto-visibility-extension timer from inside that worker thread, so the
73
+ # timer would otherwise inherit -1. A latency-sensitive timer must not
74
+ # run below normal priority: under CPU contention a delayed extension can
75
+ # miss the visibility timeout and let the message be redelivered (double
76
+ # processed). Reset to normal priority.
77
+ Thread.current.priority = 0
78
+
62
79
  until @killed
63
80
  sleep(@execution_interval)
64
81
  break if @killed