shoryuken 7.0.1 → 7.0.3

data/spec/integration/active_job/current_attributes/cross_job_reset_spec.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# A message without a CurrentAttributes key must not inherit a previous job's
+# context. Every registered class is reset after each job - even keyless ones -
+# so values set during one job don't leak into the next on a reused worker
+# thread.
+
+setup_sqs
+setup_active_job
+
+require 'active_support/current_attributes'
+require 'shoryuken/active_job/current_attributes'
+
+queue_name = DT.queue
+create_test_queue(queue_name)
+
+class LeakCurrent < ActiveSupport::CurrentAttributes
+  attribute :user_id
+end
+
+Shoryuken::ActiveJob::CurrentAttributes.persist(LeakCurrent)
+
+class CrossJobResetTestJob < ActiveJob::Base
+  def perform
+    # Record what the worker thread sees at the start of the job, then dirty
+    # Current. Without a reset after a keyless job, the second execution on the
+    # same thread would observe 'leaked' instead of nil.
+    DT[:observed] << LeakCurrent.user_id
+    LeakCurrent.user_id = 'leaked'
+  end
+end
+
+CrossJobResetTestJob.queue_as(queue_name)
+
+# Concurrency 1 so the two jobs run sequentially on the same worker thread.
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
+
+# Enqueued with an empty context, so both messages carry no cattr key.
+CrossJobResetTestJob.perform_later
+CrossJobResetTestJob.perform_later
+
+poll_queues_until(timeout: 30) { DT[:observed].size >= 2 }
+
+assert_equal([nil, nil], DT[:observed].first(2),
+             "CurrentAttributes leaked across jobs: #{DT[:observed].inspect}")

data/spec/integration/active_job/current_attributes/empty_context_spec.rb

@@ -2,7 +2,7 @@
 
 # CurrentAttributes without any values set result in nil attributes during job execution
 
-setup_localstack
+setup_sqs
 setup_active_job
 
 require 'active_support/current_attributes'

data/spec/integration/active_job/current_attributes/full_context_spec.rb

@@ -2,7 +2,7 @@
 
 # CurrentAttributes with full context are persisted and restored during job execution
 
-setup_localstack
+setup_sqs
 setup_active_job
 
 require 'active_support/current_attributes'

data/spec/integration/active_job/current_attributes/incremental_persist_spec.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# CurrentAttributes registered one class per persist call are all preserved.
+#
+# persist derived each storage key from the per-call index, so calling it once
+# per class (as an app might across several initializers) made the third call
+# reuse "cattr_0" and silently overwrite the second class - its attributes were
+# then never serialized or restored.
+
+setup_sqs
+setup_active_job
+
+require 'active_support/current_attributes'
+require 'shoryuken/active_job/current_attributes'
+
+queue_name = DT.queue
+create_test_queue(queue_name)
+
+class CurrentA < ActiveSupport::CurrentAttributes
+  attribute :a_val
+end
+
+class CurrentB < ActiveSupport::CurrentAttributes
+  attribute :b_val
+end
+
+class CurrentC < ActiveSupport::CurrentAttributes
+  attribute :c_val
+end
+
+# Registered incrementally, one class per call.
+Shoryuken::ActiveJob::CurrentAttributes.persist(CurrentA)
+Shoryuken::ActiveJob::CurrentAttributes.persist(CurrentB)
+Shoryuken::ActiveJob::CurrentAttributes.persist(CurrentC)
+
+# Every class must be registered, each under a distinct storage key.
+registered = Shoryuken::ActiveJob::CurrentAttributes.cattrs
+assert_equal(
+  %w[CurrentA CurrentB CurrentC],
+  registered.values.sort,
+  'every persisted CurrentAttributes class must be registered (none silently dropped)'
+)
+assert_equal(3, registered.keys.uniq.size, 'each class must use a distinct storage key')
+
+class IncrementalCurrentJob < ActiveJob::Base
+  def perform
+    DT[:executions] << {
+      a_val: CurrentA.a_val,
+      b_val: CurrentB.b_val,
+      c_val: CurrentC.c_val
+    }
+  end
+end
+
+IncrementalCurrentJob.queue_as(queue_name)
+
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
+
+CurrentA.a_val = 'a-value'
+CurrentB.b_val = 'b-value'
+CurrentC.c_val = 'c-value'
+
+IncrementalCurrentJob.perform_later
+
+CurrentA.reset
+CurrentB.reset
+CurrentC.reset
+
+poll_queues_until(timeout: 30) { DT[:executions].size >= 1 }
+
+result = DT[:executions].first
+assert_equal('a-value', result[:a_val])
+assert_equal('b-value', result[:b_val], 'the second incrementally-persisted class must not be dropped')
+assert_equal('c-value', result[:c_val])

data/spec/integration/active_job/current_attributes/partial_context_spec.rb

@@ -2,7 +2,7 @@
 
 # CurrentAttributes with partial values set preserve only set attributes during job execution
 
-setup_localstack
+setup_sqs
 setup_active_job
 
 require 'active_support/current_attributes'

data/spec/integration/active_job/custom_attributes/number_attributes_spec.rb

@@ -2,7 +2,7 @@
 
 # ActiveJob custom numeric message attributes are sent to SQS with correct data type
 
-setup_localstack
+setup_sqs
 setup_active_job
 
 queue_name = DT.queue

data/spec/integration/active_job/custom_attributes/string_attributes_spec.rb

@@ -2,7 +2,7 @@
 
 # ActiveJob custom string message attributes are sent to SQS and preserved
 
-setup_localstack
+setup_sqs
 setup_active_job
 
 queue_name = DT.queue

data/spec/integration/active_job/fifo_dedup_opt_out/fifo_dedup_opt_out_spec.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+# This spec tests opting out of the automatic FIFO message_deduplication_id that
+# the ActiveJob adapter generates.
+#
+# For FIFO queues the adapter derives a content-based message_deduplication_id
+# from the serialized job minus job_id/enqueued_at (see issues #457 / #750). That
+# means two *distinct* enqueues of the same job class + args within SQS's 5-minute
+# window collapse into one - the second is silently dropped. That is intentional,
+# but it is a "skipped message" trap for users who legitimately enqueue the same
+# work twice.
+#
+# Expected behavior: setting Shoryuken.active_job_fifo_message_deduplication = false
+# stops the adapter from generating the id, so those enqueues are no longer
+# silently deduplicated. The default (true) preserves the existing behavior.
+#
+# Regression: there was no way to disable the auto-generated deduplication id.
+
+setup_active_job
+
+class FifoOptOutJob < ActiveJob::Base
+  queue_as :fifo_opt_out
+
+  def perform(_arg); end
+end
+
+fifo_queue_mock = Object.new
+fifo_queue_mock.define_singleton_method(:fifo?) { true }
+fifo_queue_mock.define_singleton_method(:name) { 'fifo_opt_out.fifo' }
+
+captured = nil
+fifo_queue_mock.define_singleton_method(:send_message) { |params| captured = params }
+
+Shoryuken::Client.define_singleton_method(:queues) { |_queue_name = nil| fifo_queue_mock }
+Shoryuken.define_singleton_method(:register_worker) { |*| nil }
+
+# Default behavior: the adapter sets a content-based dedup id (excluding the
+# per-enqueue job_id/enqueued_at), so two identical jobs would collapse to one.
+FifoOptOutJob.perform_later('same-args')
+
+assert(
+  captured.key?(:message_deduplication_id),
+  'by default a FIFO ActiveJob send gets an auto-generated message_deduplication_id'
+)
+
+body = captured[:message_body]
+expected = Digest::SHA256.hexdigest(JSON.dump(body.except('job_id', 'enqueued_at')))
+assert_equal(expected, captured[:message_deduplication_id])
+
+# Opt-out: with deduplication disabled the adapter no longer sets a dedup id, so
+# distinct enqueues of identical jobs are not silently dropped.
+Shoryuken.active_job_fifo_message_deduplication = false
+captured = nil
+FifoOptOutJob.perform_later('same-args')
+
+refute(
+  captured.key?(:message_deduplication_id),
+  'with deduplication disabled the adapter must not set message_deduplication_id'
+)
+
+# An explicit deduplication id must still be honored even when auto-generation is off.
+captured = nil
+FifoOptOutJob.set(message_deduplication_id: 'explicit-id').perform_later('same-args')
+
+assert_equal('explicit-id', captured[:message_deduplication_id])

data/spec/integration/active_job/keyword_arguments/keyword_arguments_spec.rb

@@ -6,7 +6,7 @@
 # In Shoryuken 7.0, the SQSSendMessageParametersSupport module's initialize method
 # breaks keyword argument passing to ActiveJob jobs because it lacks ruby2_keywords.
 
-setup_localstack
+setup_sqs
 setup_active_job
 
 queue_name = DT.queue

data/spec/integration/active_job/retry/discard_on_spec.rb

@@ -2,7 +2,7 @@
 
 # ActiveJob discard_on discards jobs that raise specific errors without retry
 
-setup_localstack
+setup_sqs
 setup_active_job
 
 queue_name = DT.queue

data/spec/integration/active_job/retry/retry_on_spec.rb

@@ -2,7 +2,7 @@
 
 # ActiveJob retry_on re-enqueues failed jobs until they succeed or exhaust attempts
 
-setup_localstack
+setup_sqs
 setup_active_job
 
 queue_name = DT.queue

data/spec/integration/active_job/roundtrip/roundtrip_spec.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
 # Full round-trip ActiveJob integration test
-# Enqueues a job via ActiveJob → sends to LocalStack SQS → processes via Shoryuken → verifies execution
+# Enqueues a job via ActiveJob → sends to ElasticMQ SQS → processes via Shoryuken → verifies execution
 
-setup_localstack
+setup_sqs
 setup_active_job
 
 queue_name = DT.queue

data/spec/integration/active_job/scheduled/scheduled_spec.rb

@@ -3,7 +3,7 @@
 # Scheduled ActiveJob integration test
 # Tests jobs scheduled with set(wait:) are delivered after the delay
 
-setup_localstack
+setup_sqs
 setup_active_job
 
 queue_name = DT.queue

data/spec/integration/active_record_middleware/active_record_middleware_spec.rb

@@ -3,7 +3,7 @@
 # This spec tests the ActiveRecord middleware functionality.
 # The middleware clears database connections after each message is processed.
 
-setup_localstack
+setup_sqs
 
 queue_name = DT.queue
 create_test_queue(queue_name)

data/spec/integration/auto_delete/auto_delete_spec.rb

@@ -3,7 +3,7 @@
 # This spec tests the auto_delete middleware functionality.
 # When auto_delete: true, messages should be automatically deleted after successful processing.
 
-setup_localstack
+setup_sqs
 
 queue_name = DT.queue
 create_test_queue(queue_name)

data/spec/integration/auto_extend_visibility/auto_extend_visibility_spec.rb

@@ -4,7 +4,7 @@
 # When auto_visibility_timeout: true, the message visibility timeout should be
 # automatically extended during long-running job processing to prevent re-delivery.
 
-setup_localstack
+setup_sqs
 
 queue_name = DT.uuid
 

data/spec/integration/auto_extend_visibility/short_visibility_timeout_spec.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This spec tests that auto_visibility_timeout still processes messages when the
+# queue's visibility timeout is short (<= EXTEND_UPFRONT_SECONDS).
+#
+# AutoExtendVisibility schedules a TimerTask at execution_interval =
+# visibility_timeout - EXTEND_UPFRONT_SECONDS (5s). When the queue's visibility
+# timeout is <= 5s that interval is <= 0, and TimerTask#initialize raises
+# ArgumentError *before* the worker runs - so every receive fails and the
+# message is reprocessed (and re-fails) until it hits a DLQ.
+#
+# Expected behavior: a short visibility timeout no longer breaks processing; the
+# extension interval is clamped to a positive value and the worker runs normally.
+#
+# Regression: auto_visibility_timeout + a <= 5s queue visibility timeout broke
+# processing entirely.
+
+require 'timeout'
+
+setup_sqs
+
+DT.clear
+
+queue_name = DT.queue
+# 5s visibility timeout -> interval would be 5 - 5 = 0 -> TimerTask raised pre-fix.
+create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '5' })
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+
+short_vt_worker = Class.new do
+  include Shoryuken::Worker
+
+  shoryuken_options auto_delete: true, auto_visibility_timeout: true
+
+  def perform(_sqs_msg, body)
+    DT[:processed] << body
+  end
+end
+
+short_vt_worker.get_shoryuken_options['queue'] = queue_name
+Shoryuken.register_worker(queue_name, short_vt_worker)
+
+queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
+Shoryuken::Client.sqs.send_message(queue_url: queue_url, message_body: 'hello')
+
+poll_queues_until(timeout: 20) { DT[:processed].size >= 1 }
+
+assert_equal(
+  ['hello'],
+  DT[:processed].to_a,
+  'auto_visibility_timeout must not break processing on a queue with a short visibility timeout'
+)

data/spec/integration/aws_config/aws_config_spec.rb

@@ -10,7 +10,7 @@ Shoryuken.sqs_client = nil
 # Configure AWS options programmatically (simulating PORO setup with config file)
 Shoryuken.options[:aws] = {
   region: 'us-east-1',
-  endpoint: 'http://localhost:4566',
+  endpoint: 'http://localhost:9324',
   access_key_id: 'test-key-from-config',
   secret_access_key: 'test-secret-from-config'
 }
@@ -22,7 +22,7 @@ client = Shoryuken.sqs_client
 config = client.config
 
 assert_equal('us-east-1', config.region, "Region should be set from options[:aws]")
-assert_equal('http://localhost:4566', config.endpoint.to_s, "Endpoint should be set from options[:aws]")
+assert_equal('http://localhost:9324', config.endpoint.to_s, "Endpoint should be set from options[:aws]")
 
 # Verify the client actually works by creating a queue
 queue_name = "aws-config-test-#{SecureRandom.hex(6)}"
@@ -49,7 +49,7 @@ assert_equal('us-west-2', client2.config.region, "New client should use updated
 Shoryuken.sqs_client = nil
 Shoryuken.options[:aws] = {
   region: 'us-east-1',
-  endpoint: 'http://localhost:4566',
+  endpoint: 'http://localhost:9324',
   access_key_id: 'another-key',
   secret_access_key: 'another-secret'
 }

data/spec/integration/batch_processing/batch_processing_spec.rb

@@ -4,7 +4,7 @@
 # messages), batch vs single worker behavior differences, JSON body parsing in
 # batch mode, and maximum batch size handling.
 
-setup_localstack
+setup_sqs
 
 queue_name = DT.queue
 create_test_queue(queue_name)

data/spec/integration/body_parser/json_parser_spec.rb

@@ -3,7 +3,7 @@
 # This spec tests the body_parser option with :json setting
 # Verifies that JSON messages are automatically parsed into Ruby hashes
 
-setup_localstack
+setup_sqs
 
 queue_name = DT.uuid
 create_test_queue(queue_name)

data/spec/integration/body_parser/proc_parser_spec.rb

@@ -3,7 +3,7 @@
 # This spec tests the body_parser option with a custom Proc
 # Verifies that custom parsing logic can be applied to messages
 
-setup_localstack
+setup_sqs
 
 queue_name = DT.uuid
 create_test_queue(queue_name)

data/spec/integration/body_parser/text_parser_spec.rb

@@ -3,7 +3,7 @@
 # This spec tests the body_parser option with :text setting (default)
 # Verifies that messages are returned as plain strings
 
-setup_localstack
+setup_sqs
 
 queue_name = DT.uuid
 create_test_queue(queue_name)

data/spec/integration/concurrent_processing/concurrent_processing_spec.rb

@@ -4,7 +4,7 @@
 
 require 'concurrent'
 
-setup_localstack
+setup_sqs
 
 queue_name = DT.queue
 create_test_queue(queue_name)

data/spec/integration/concurrent_processing/processor_accounting_spec.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+# This spec tests that the manager's busy-processor accounting stays correct
+# when processor completion handling itself raises.
+#
+# Manager#processor_done makes SQS calls (Shoryuken::Client.queues, #fifo?)
+# and invokes the polling strategy's message_processed callback (a documented
+# extension point) - any of these can raise, e.g. on a transient network
+# error or a bug in a custom strategy.
+#
+# Expected behavior: processor_done runs exactly once per processed message
+# and busy_processors never goes negative, even when processor_done raises.
+#
+# Regression: Manager#assign chained `.then { processor_done }` with
+# `.rescue { processor_done }`, so an exception inside processor_done
+# rejected the then-promise and ran processor_done a SECOND time. The busy
+# counter was decremented twice for one message and drifted negative,
+# inflating `ready` and silently breaking the concurrency limit for the
+# life of the process.
+
+require 'timeout'
+
+setup_sqs
+
+DT.clear
+
+queue_name = "#{DT.uuid}.fifo"
+create_fifo_queue(queue_name)
+
+# A custom polling strategy whose message_processed callback raises, like a
+# buggy user strategy or one that performs I/O and hits a transient error.
+# Manager#processor_done invokes it for FIFO queues.
+class FlakyCallbackStrategy < Shoryuken::Polling::WeightedRoundRobin
+  def message_processed(queue)
+    DT[:message_processed_calls] << { queue: queue, time: Time.now }
+    raise 'flaky message_processed callback'
+  end
+end
+
+Shoryuken.add_group('default', 2, polling_strategy: FlakyCallbackStrategy)
+Shoryuken.add_queue(queue_name, 1, 'default')
+
+accounting_worker = Class.new do
+  include Shoryuken::Worker
+
+  shoryuken_options auto_delete: true
+
+  def perform(_sqs_msg, body)
+    DT[:processed] << body
+  end
+end
+
+accounting_worker.get_shoryuken_options['queue'] = queue_name
+Shoryuken.register_worker(queue_name, accounting_worker)
+
+# busy_processors is exposed through the public utilization_update event
+Shoryuken.on(:utilization_update) do |opts|
+  DT[:utilization] << opts.dup
+end
+
+launcher = Shoryuken::Launcher.new
+launcher.start
+
+begin
+  Shoryuken::Client.queues(queue_name).send_message(message_body: 'hello')
+
+  # Wait for the message to be processed and the completion path to run
+  Timeout.timeout(15) { sleep 0.5 until DT[:message_processed_calls].size >= 1 }
+
+  # Give the (buggy) rescue path time to run processor_done a second time
+  sleep 3
+
+  assert_equal(['hello'], DT[:processed].to_a, 'Message should have been processed exactly once')
+
+  assert_equal(
+    1,
+    DT[:message_processed_calls].size,
+    'processor_done must run exactly once per message, even when it raises ' \
+    "(message_processed was called #{DT[:message_processed_calls].size} times)"
+  )
+
+  negative = DT[:utilization].select { |u| u[:busy_processors].negative? }
+  assert(
+    negative.empty?,
+    'busy_processors must never go negative; a double decrement inflates ready ' \
+    "and breaks the concurrency limit (saw: #{negative.first.inspect})"
+  )
+ensure
+  begin
+    Timeout.timeout(10) { launcher.stop }
+  rescue Timeout::Error
+    nil
+  end
+end

data/spec/integration/custom_group_polling_strategy/custom_group_polling_strategy_spec.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+# This spec tests that a custom polling strategy can be configured per-group
+# via the add_group API.
+#
+# Bug #925: Cannot configure custom polling_strategy from YAML config
+# The add_group method does not accept a polling_strategy parameter,
+# making it impossible to set a per-group polling strategy programmatically.
+# Additionally, polling_strategy() reads from options[:groups] (raw config)
+# rather than from groups (the processed hash populated by add_group).
+
+# Define a custom polling strategy
+class CustomRoundRobin < Shoryuken::Polling::BaseStrategy
+  def initialize(queues, delay = nil)
+    @queues = queues.dup.uniq
+    @delay = delay
+    @index = 0
+  end
+
+  def next_queue
+    return nil if @queues.empty?
+
+    queue = @queues[@index % @queues.length]
+    @index += 1
+    Shoryuken::Polling::QueueConfiguration.new(queue, {})
+  end
+
+  def messages_found(_queue, _count)
+    # noop
+  end
+
+  def active_queues
+    @queues.map { |q| [q, 1] }
+  end
+end
+
+# ---- Part 1: API assertions (no SQS needed) ----
+# add_group accepts polling_strategy: keyword argument
+Shoryuken.add_group('custom_group', 1, polling_strategy: CustomRoundRobin)
+
+# polling_strategy() returns our custom strategy for the group
+strategy = Shoryuken.polling_strategy('custom_group')
+assert_equal(
+  CustomRoundRobin,
+  strategy,
+  "Expected polling_strategy('custom_group') to return CustomRoundRobin, got #{strategy.inspect}"
+)
+
+# add_group raises InvalidPollingStrategyError for invalid types
+begin
+  Shoryuken.add_group('bad_group', 1, polling_strategy: 123)
+  raise 'Expected InvalidPollingStrategyError to be raised'
+rescue Shoryuken::Errors::InvalidPollingStrategyError
+  # expected
+end
+
+# ---- Part 2: End-to-end with SQS ----
+setup_sqs
+
+queue_name = DT.queues[0]
+create_test_queue(queue_name)
+
+Shoryuken.groups.clear
+Shoryuken.add_group('custom_group', 1, polling_strategy: CustomRoundRobin)
+Shoryuken.add_queue(queue_name, 1, 'custom_group')
+
+worker_class = Class.new do
+  include Shoryuken::Worker
+
+  shoryuken_options auto_delete: true, batch: false
+
+  def perform(_sqs_msg, body)
+    DT[:processed] << body
+  end
+end
+
+worker_class.get_shoryuken_options['queue'] = queue_name
+Shoryuken.register_worker(queue_name, worker_class)
+
+Shoryuken::Client.queues(queue_name).send_message(message_body: 'custom-strategy-msg')
+
+sleep 1
+
+poll_queues_until { DT[:processed].size >= 1 }
+
+assert_equal(1, DT[:processed].size)
+assert_equal('custom-strategy-msg', DT[:processed].first)

data/spec/integration/dead_letter_queue/dead_letter_queue_spec.rb

@@ -5,7 +5,7 @@
 # Note: This test doesn't use poll_queues_until because messages should fail
 # and be moved to DLQ rather than being successfully processed.
 
-setup_localstack
+setup_sqs
 
 main_queue_name = DT.queues[0]
 dlq_name = DT.queues[1]

data/spec/integration/exception_handlers/exception_handlers_spec.rb

@@ -3,7 +3,7 @@
 # This spec tests custom exception handlers.
 # Exception handlers are called when a worker raises an error.
 
-setup_localstack
+setup_sqs
 
 queue_name = DT.queue
 create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '2' })

data/spec/integration/exponential_backoff/exponential_backoff_spec.rb

@@ -4,7 +4,7 @@
 # When retry_intervals is configured, failed jobs should have their visibility
 # timeout adjusted based on the retry attempt number.
 
-setup_localstack
+setup_sqs
 
 queue_name = DT.queue