shoryuken 6.2.1 → 7.0.2

data/spec/integration/active_job/scheduled/scheduled_spec.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+# Scheduled ActiveJob integration test
+# Tests jobs scheduled with set(wait:) are delivered after the delay
+
+setup_sqs
+setup_active_job
+
+queue_name = DT.queue
+create_test_queue(queue_name)
+
+class ScheduledTestJob < ActiveJob::Base
+  def perform(label)
+    DT[:executions] << {
+      label: label,
+      job_id: job_id,
+      executed_at: Time.now
+    }
+  end
+end
+
+ScheduledTestJob.queue_as(queue_name)
+
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+Shoryuken.register_worker(queue_name, Shoryuken::ActiveJob::JobWrapper)
+
+immediate_enqueue_time = Time.now
+ScheduledTestJob.perform_later('immediate')
+DT[:timestamps] << { label: 'immediate', time: immediate_enqueue_time }
+
+delayed_enqueue_time = Time.now
+ScheduledTestJob.set(wait: 3.seconds).perform_later('delayed_3s')
+DT[:timestamps] << { label: 'delayed_3s', time: delayed_enqueue_time }
+
+delayed_5s_enqueue_time = Time.now
+ScheduledTestJob.set(wait: 5.seconds).perform_later('delayed_5s')
+DT[:timestamps] << { label: 'delayed_5s', time: delayed_5s_enqueue_time }
+
+poll_queues_until(timeout: 30) do
+  DT[:executions].size >= 3
+end
+
+assert_equal(3, DT[:executions].size, "Expected 3 job executions")
+
+# Find each job's execution
+immediate_job = DT[:executions].find { |e| e[:label] == 'immediate' }
+delayed_3s_job = DT[:executions].find { |e| e[:label] == 'delayed_3s' }
+delayed_5s_job = DT[:executions].find { |e| e[:label] == 'delayed_5s' }
+
+assert(immediate_job, "Immediate job should have executed")
+assert(delayed_3s_job, "3s delayed job should have executed")
+assert(delayed_5s_job, "5s delayed job should have executed")
+
+def enqueue_time(label)
+  DT[:timestamps].find { |t| t[:label] == label }[:time]
+end
+
+immediate_delay = immediate_job[:executed_at] - enqueue_time('immediate')
+assert(immediate_delay < 10, "Immediate job should execute within 10 seconds, took #{immediate_delay}s")
+
+# Using 2 seconds tolerance for SQS delivery variation
+delayed_3s_actual_delay = delayed_3s_job[:executed_at] - enqueue_time('delayed_3s')
+assert(delayed_3s_actual_delay >= 2, "3s delayed job should execute after at least 2s, took #{delayed_3s_actual_delay}s")
+
+delayed_5s_actual_delay = delayed_5s_job[:executed_at] - enqueue_time('delayed_5s')
+assert(delayed_5s_actual_delay >= 4, "5s delayed job should execute after at least 4s, took #{delayed_5s_actual_delay}s")
+
+assert(
+  immediate_job[:executed_at] <= delayed_3s_job[:executed_at],
+  "Immediate job should execute before 3s delayed job"
+)
+assert(
+  delayed_3s_job[:executed_at] <= delayed_5s_job[:executed_at],
+  "3s delayed job should execute before 5s delayed job"
+)

data/spec/integration/active_record_middleware/active_record_middleware_spec.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+# This spec tests the ActiveRecord middleware functionality.
+# The middleware clears database connections after each message is processed.
+
+setup_sqs
+
+queue_name = DT.queue
+create_test_queue(queue_name)
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+
+# Mock ActiveRecord module to track connection clearing
+module ActiveRecord
+  VERSION = Gem::Version.new('7.2.0')
+
+  def self.version
+    VERSION
+  end
+
+  class Base
+    class << self
+      attr_accessor :connections_cleared
+
+      def connection_handler
+        @connection_handler ||= ConnectionHandler.new
+      end
+    end
+
+    self.connections_cleared = []
+  end
+
+  class ConnectionHandler
+    def clear_active_connections!(scope)
+      ActiveRecord::Base.connections_cleared << { scope: scope, time: Time.now }
+    end
+  end
+end
+
+# Add the ActiveRecord middleware to the chain
+require 'shoryuken/middleware/server/active_record'
+Shoryuken.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Shoryuken::Middleware::Server::ActiveRecord
+  end
+end
+
+worker_class = Class.new do
+  include Shoryuken::Worker
+
+  shoryuken_options auto_delete: true, batch: false
+
+  def perform(sqs_msg, body)
+    DT[:processed] << { message_id: sqs_msg.message_id, body: body }
+  end
+end
+
+worker_class.get_shoryuken_options['queue'] = queue_name
+Shoryuken.register_worker(queue_name, worker_class)
+
+# Clear any prior connection clearing records
+ActiveRecord::Base.connections_cleared.clear
+
+# Send multiple messages
+3.times { |i| Shoryuken::Client.queues(queue_name).send_message(message_body: "ar-test-#{i}") }
+
+sleep 1
+
+poll_queues_until { DT[:processed].size >= 3 }
+
+# Verify all messages were processed
+assert_equal(3, DT[:processed].size)
+
+# Verify ActiveRecord connections were cleared after each message
+# The middleware should have called clear_active_connections! for each message
+assert(
+  ActiveRecord::Base.connections_cleared.size >= 3,
+  "ActiveRecord connections should be cleared after each message (cleared #{ActiveRecord::Base.connections_cleared.size} times)"
+)
+
+# Verify the :all scope was used (Rails 7.1+ behavior)
+ActiveRecord::Base.connections_cleared.each do |record|
+  assert_equal(:all, record[:scope], 'Should use :all scope for Rails 7.1+')
+end

data/spec/integration/auto_delete/auto_delete_spec.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# This spec tests the auto_delete middleware functionality.
+# When auto_delete: true, messages should be automatically deleted after successful processing.
+
+setup_sqs
+
+queue_name = DT.queue
+create_test_queue(queue_name)
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+
+auto_delete_worker = Class.new do
+  include Shoryuken::Worker
+
+  shoryuken_options auto_delete: true
+
+  def perform(sqs_msg, body)
+    DT[:auto_delete_processed] << { message_id: sqs_msg.message_id, body: body }
+  end
+end
+
+auto_delete_worker.get_shoryuken_options['queue'] = queue_name
+Shoryuken.register_worker(queue_name, auto_delete_worker)
+
+queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
+
+# Send a message
+Shoryuken::Client.sqs.send_message(
+  queue_url: queue_url,
+  message_body: 'auto delete test'
+)
+
+sleep 1
+
+# Process the message
+poll_queues_until { DT[:auto_delete_processed].size >= 1 }
+
+assert_equal(1, DT[:auto_delete_processed].size)
+assert_equal('auto delete test', DT[:auto_delete_processed].first[:body])
+
+# Wait a moment for deletion to complete
+sleep 2
+
+# Verify message was deleted - queue should be empty
+attributes = Shoryuken::Client.sqs.get_queue_attributes(
+  queue_url: queue_url,
+  attribute_names: ['ApproximateNumberOfMessages', 'ApproximateNumberOfMessagesNotVisible']
+).attributes
+
+total_messages = attributes['ApproximateNumberOfMessages'].to_i +
+                 attributes['ApproximateNumberOfMessagesNotVisible'].to_i
+assert_equal(0, total_messages, "Message should be deleted when auto_delete: true")

data/spec/integration/auto_extend_visibility/auto_extend_visibility_spec.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# This spec tests the auto_visibility_timeout middleware functionality.
+# When auto_visibility_timeout: true, the message visibility timeout should be
+# automatically extended during long-running job processing to prevent re-delivery.
+
+setup_sqs
+
+queue_name = DT.uuid
+
+# Create queue with short visibility timeout (10 seconds)
+create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '10' })
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+
+# Worker with auto_visibility_timeout enabled that takes longer than visibility timeout
+auto_visibility_worker = Class.new do
+  include Shoryuken::Worker
+
+  shoryuken_options auto_visibility_timeout: true, auto_delete: true
+
+  def perform(sqs_msg, body)
+    DT[:processing_started] << Time.now
+    # Sleep longer than the queue's visibility timeout (10s)
+    # The middleware should extend visibility before it expires
+    sleep 12
+    DT[:processing_completed] << Time.now
+    DT[:processed_messages] << { message_id: sqs_msg.message_id, body: body }
+  end
+end
+
+auto_visibility_worker.get_shoryuken_options['queue'] = queue_name
+Shoryuken.register_worker(queue_name, auto_visibility_worker)
+
+queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
+
+# Send a message
+Shoryuken::Client.sqs.send_message(
+  queue_url: queue_url,
+  message_body: 'long running job'
+)
+
+sleep 1
+
+# Process the message - this should take ~12 seconds but not fail
+poll_queues_until(timeout: 30) { DT[:processed_messages].size >= 1 }
+
+# Verify message was processed exactly once (visibility was extended, not re-delivered)
+assert_equal(1, DT[:processed_messages].size, "Message should be processed exactly once")
+assert_equal('long running job', DT[:processed_messages].first[:body])
+
+# Verify processing took longer than the visibility timeout
+processing_time = DT[:processing_completed].first - DT[:processing_started].first
+assert(processing_time >= 12, "Processing should have taken at least 12 seconds")
+
+# Cleanup
+delete_test_queue(queue_name)

data/spec/integration/aws_config/aws_config_spec.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+# This spec tests that AWS configuration from Shoryuken.options[:aws]
+# is properly passed to the SQS client initialization.
+# This verifies the fix for issue #815: PORO setup does not load AWS config
+
+# Reset any cached SQS client to ensure fresh initialization
+Shoryuken.sqs_client = nil
+
+# Configure AWS options programmatically (simulating PORO setup with config file)
+Shoryuken.options[:aws] = {
+  region: 'us-east-1',
+  endpoint: 'http://localhost:9324',
+  access_key_id: 'test-key-from-config',
+  secret_access_key: 'test-secret-from-config'
+}
+
+# Get the SQS client - this should use the AWS config from options
+client = Shoryuken.sqs_client
+
+# Verify the client was configured with our options
+config = client.config
+
+assert_equal('us-east-1', config.region, "Region should be set from options[:aws]")
+assert_equal('http://localhost:9324', config.endpoint.to_s, "Endpoint should be set from options[:aws]")
+
+# Verify the client actually works by creating a queue
+queue_name = "aws-config-test-#{SecureRandom.hex(6)}"
+
+begin
+  result = client.create_queue(queue_name: queue_name)
+  assert(result.queue_url.include?(queue_name), "Should be able to create queue with configured client")
+
+  # Clean up
+  client.delete_queue(queue_url: result.queue_url)
+rescue Aws::SQS::Errors::ServiceError => e
+  raise "SQS client should work with configured AWS options: #{e.message}"
+end
+
+# Test 2: Verify that reconfiguring options and resetting client works
+Shoryuken.sqs_client = nil
+Shoryuken.options[:aws][:region] = 'us-west-2'
+
+client2 = Shoryuken.sqs_client
+assert_equal('us-west-2', client2.config.region, "New client should use updated region")
+
+# Test 3: Verify that credentials from options are used
+# Reset and reconfigure with explicit credentials
+Shoryuken.sqs_client = nil
+Shoryuken.options[:aws] = {
+  region: 'us-east-1',
+  endpoint: 'http://localhost:9324',
+  access_key_id: 'another-key',
+  secret_access_key: 'another-secret'
+}
+
+client3 = Shoryuken.sqs_client
+assert(client3.is_a?(Aws::SQS::Client), "Client should be created with new credentials")
+assert_equal('us-east-1', client3.config.region, "Region should match configured value")

data/spec/integration/batch_processing/batch_processing_spec.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This spec tests batch processing including batch message reception (up to 10
+# messages), batch vs single worker behavior differences, JSON body parsing in
+# batch mode, and maximum batch size handling.
+
+setup_sqs
+
+queue_name = DT.queue
+create_test_queue(queue_name)
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+
+worker_class = Class.new do
+  include Shoryuken::Worker
+
+  def perform(sqs_msgs, bodies)
+    msgs = Array(sqs_msgs)
+    DT[:batch_sizes] << msgs.size
+    DT[:messages].concat(Array(bodies))
+  end
+end
+
+worker_class.get_shoryuken_options['queue'] = queue_name
+worker_class.get_shoryuken_options['auto_delete'] = true
+worker_class.get_shoryuken_options['batch'] = true
+Shoryuken.register_worker(queue_name, worker_class)
+
+entries = 5.times.map { |i| { id: SecureRandom.uuid, message_body: "message-#{i}" } }
+Shoryuken::Client.queues(queue_name).send_messages(entries: entries)
+
+sleep 1
+
+poll_queues_until { DT[:messages].size >= 5 }
+
+assert_equal(5, DT[:messages].size)
+assert(DT[:batch_sizes].any? { |size| size > 1 }, "Expected at least one batch with size > 1")

data/spec/integration/body_parser/json_parser_spec.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# This spec tests the body_parser option with :json setting
+# Verifies that JSON messages are automatically parsed into Ruby hashes
+
+setup_sqs
+
+queue_name = DT.uuid
+create_test_queue(queue_name)
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+
+worker_class = Class.new do
+  include Shoryuken::Worker
+
+  shoryuken_options body_parser: :json
+
+  def perform(sqs_msg, body)
+    DT[:parsed_bodies] << body
+    DT[:body_classes] << body.class.name
+  end
+end
+
+worker_class.get_shoryuken_options['queue'] = queue_name
+worker_class.get_shoryuken_options['auto_delete'] = true
+Shoryuken.register_worker(queue_name, worker_class)
+
+queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
+
+# Send a JSON message
+json_body = { 'name' => 'test', 'value' => 42, 'nested' => { 'key' => 'val' } }
+Shoryuken::Client.sqs.send_message(
+  queue_url: queue_url,
+  message_body: JSON.dump(json_body)
+)
+
+sleep 1
+
+poll_queues_until { DT[:parsed_bodies].size >= 1 }
+
+assert_equal(1, DT[:parsed_bodies].size)
+assert_equal('Hash', DT[:body_classes].first)
+assert_equal('test', DT[:parsed_bodies].first['name'])
+assert_equal(42, DT[:parsed_bodies].first['value'])
+assert_equal('val', DT[:parsed_bodies].first['nested']['key'])

data/spec/integration/body_parser/proc_parser_spec.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# This spec tests the body_parser option with a custom Proc
+# Verifies that custom parsing logic can be applied to messages
+
+setup_sqs
+
+queue_name = DT.uuid
+create_test_queue(queue_name)
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+
+# Custom parser that uppercases the body and adds metadata
+custom_parser = proc do |sqs_msg|
+  {
+    original: sqs_msg.body,
+    transformed: sqs_msg.body.upcase,
+    message_id: sqs_msg.message_id
+  }
+end
+
+worker_class = Class.new do
+  include Shoryuken::Worker
+
+  def perform(sqs_msg, body)
+    DT[:parsed_bodies] << body
+    DT[:body_classes] << body.class.name
+  end
+end
+
+worker_class.get_shoryuken_options['queue'] = queue_name
+worker_class.get_shoryuken_options['auto_delete'] = true
+worker_class.get_shoryuken_options['body_parser'] = custom_parser
+Shoryuken.register_worker(queue_name, worker_class)
+
+queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
+
+# Send a message to be processed by custom parser
+Shoryuken::Client.sqs.send_message(
+  queue_url: queue_url,
+  message_body: 'hello world'
+)
+
+sleep 1
+
+poll_queues_until { DT[:parsed_bodies].size >= 1 }
+
+assert_equal(1, DT[:parsed_bodies].size)
+assert_equal('Hash', DT[:body_classes].first)
+
+parsed = DT[:parsed_bodies].first
+assert_equal('hello world', parsed[:original])
+assert_equal('HELLO WORLD', parsed[:transformed])
+assert(parsed[:message_id], "Should include message_id from sqs_msg")

data/spec/integration/body_parser/text_parser_spec.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# This spec tests the body_parser option with :text setting (default)
+# Verifies that messages are returned as plain strings
+
+setup_sqs
+
+queue_name = DT.uuid
+create_test_queue(queue_name)
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+
+worker_class = Class.new do
+  include Shoryuken::Worker
+
+  shoryuken_options body_parser: :text
+
+  def perform(sqs_msg, body)
+    DT[:parsed_bodies] << body
+    DT[:body_classes] << body.class.name
+  end
+end
+
+worker_class.get_shoryuken_options['queue'] = queue_name
+worker_class.get_shoryuken_options['auto_delete'] = true
+Shoryuken.register_worker(queue_name, worker_class)
+
+queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
+
+# Send a plain text message
+text_body = 'Hello, this is a plain text message!'
+Shoryuken::Client.sqs.send_message(
+  queue_url: queue_url,
+  message_body: text_body
+)
+
+sleep 1
+
+poll_queues_until { DT[:parsed_bodies].size >= 1 }
+
+assert_equal(1, DT[:parsed_bodies].size)
+assert_equal('String', DT[:body_classes].first)
+assert_equal(text_body, DT[:parsed_bodies].first)

data/spec/integration/concurrent_processing/concurrent_processing_spec.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# This spec tests concurrent message processing with multiple processors.
+
+require 'concurrent'
+
+setup_sqs
+
+queue_name = DT.queue
+create_test_queue(queue_name)
+Shoryuken.add_group('concurrent', 5) # 5 concurrent processors
+Shoryuken.add_queue(queue_name, 1, 'concurrent')
+
+# Atomic counters for tracking concurrency
+concurrent_count = Concurrent::AtomicFixnum.new(0)
+max_concurrent = Concurrent::AtomicFixnum.new(0)
+
+worker_class = Class.new do
+  include Shoryuken::Worker
+
+  shoryuken_options auto_delete: true, batch: false
+
+  define_method(:perform) do |sqs_msg, body|
+    concurrent_count.increment
+    current = concurrent_count.value
+    max_concurrent.update { |max| [max, current].max }
+
+    sleep 0.5 # Simulate work
+
+    DT[:processing_times] << Time.now
+
+    concurrent_count.decrement
+  end
+end
+
+worker_class.get_shoryuken_options['queue'] = queue_name
+Shoryuken.register_worker(queue_name, worker_class)
+
+10.times { |i| Shoryuken::Client.queues(queue_name).send_message(message_body: "msg-#{i}") }
+
+poll_queues_until(timeout: 20) { DT[:processing_times].size >= 10 }
+
+assert_equal(10, DT[:processing_times].size)
+# With multiple processors, we should see concurrency > 1
+assert(max_concurrent.value > 1, "Expected concurrency > 1, got #{max_concurrent.value}")

data/spec/integration/custom_group_polling_strategy/custom_group_polling_strategy_spec.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+# This spec tests that a custom polling strategy can be configured per-group
+# via the add_group API.
+#
+# Bug #925: Cannot configure custom polling_strategy from YAML config
+# The add_group method does not accept a polling_strategy parameter,
+# making it impossible to set a per-group polling strategy programmatically.
+# Additionally, polling_strategy() reads from options[:groups] (raw config)
+# rather than from groups (the processed hash populated by add_group).
+
+# Define a custom polling strategy
+class CustomRoundRobin < Shoryuken::Polling::BaseStrategy
+  def initialize(queues, delay = nil)
+    @queues = queues.dup.uniq
+    @delay = delay
+    @index = 0
+  end
+
+  def next_queue
+    return nil if @queues.empty?
+
+    queue = @queues[@index % @queues.length]
+    @index += 1
+    Shoryuken::Polling::QueueConfiguration.new(queue, {})
+  end
+
+  def messages_found(_queue, _count)
+    # noop
+  end
+
+  def active_queues
+    @queues.map { |q| [q, 1] }
+  end
+end
+
+# ---- Part 1: API assertions (no SQS needed) ----
+# add_group accepts polling_strategy: keyword argument
+Shoryuken.add_group('custom_group', 1, polling_strategy: CustomRoundRobin)
+
+# polling_strategy() returns our custom strategy for the group
+strategy = Shoryuken.polling_strategy('custom_group')
+assert_equal(
+  CustomRoundRobin,
+  strategy,
+  "Expected polling_strategy('custom_group') to return CustomRoundRobin, got #{strategy.inspect}"
+)
+
+# add_group raises InvalidPollingStrategyError for invalid types
+begin
+  Shoryuken.add_group('bad_group', 1, polling_strategy: 123)
+  raise 'Expected InvalidPollingStrategyError to be raised'
+rescue Shoryuken::Errors::InvalidPollingStrategyError
+  # expected
+end
+
+# ---- Part 2: End-to-end with SQS ----
+setup_sqs
+
+queue_name = DT.queues[0]
+create_test_queue(queue_name)
+
+Shoryuken.groups.clear
+Shoryuken.add_group('custom_group', 1, polling_strategy: CustomRoundRobin)
+Shoryuken.add_queue(queue_name, 1, 'custom_group')
+
+worker_class = Class.new do
+  include Shoryuken::Worker
+
+  shoryuken_options auto_delete: true, batch: false
+
+  def perform(_sqs_msg, body)
+    DT[:processed] << body
+  end
+end
+
+worker_class.get_shoryuken_options['queue'] = queue_name
+Shoryuken.register_worker(queue_name, worker_class)
+
+Shoryuken::Client.queues(queue_name).send_message(message_body: 'custom-strategy-msg')
+
+sleep 1
+
+poll_queues_until { DT[:processed].size >= 1 }
+
+assert_equal(1, DT[:processed].size)
+assert_equal('custom-strategy-msg', DT[:processed].first)