shoryuken 6.2.1 → 7.0.2

data/spec/integration/rails/rails_81/continuation_spec.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# ActiveJob Continuations integration tests for Rails 8.1+
+# Tests the stopping? method and continuation timestamp handling
+
+setup_active_job
+
+# Skip if ActiveJob::Continuable is not available (Rails < 8.1)
+unless defined?(ActiveJob::Continuable)
+  puts "Skipping continuation tests - ActiveJob::Continuable not available (requires Rails 8.1+)"
+  exit 0
+end
+
+# Test stopping? returns false when launcher is not initialized
+adapter = ActiveJob::QueueAdapters::ShoryukenAdapter.new
+assert_equal(false, adapter.stopping?)
+
+# Test stopping? returns true when launcher is stopping
+launcher = Shoryuken::Launcher.new
+runner = Shoryuken::Runner.instance
+runner.instance_variable_set(:@launcher, launcher)
+
+adapter2 = ActiveJob::QueueAdapters::ShoryukenAdapter.new
+assert_equal(false, adapter2.stopping?)
+
+launcher.instance_variable_set(:@stopping, true)
+assert_equal(true, adapter2.stopping?)
+
+# Reset launcher state
+launcher.instance_variable_set(:@stopping, false)
+
+# Test past timestamps for continuation retries
+job_capture = JobCapture.new
+job_capture.start_capturing
+
+class ContinuableTestJob < ActiveJob::Base
+  include ActiveJob::Continuable if defined?(ActiveJob::Continuable)
+  queue_as :default
+  def perform; end
+end
+
+adapter3 = ActiveJob::QueueAdapters::ShoryukenAdapter.new
+job = ContinuableTestJob.new
+job.sqs_send_message_parameters = {}
+
+past_timestamp = Time.current.to_f - 60
+adapter3.enqueue_at(job, past_timestamp)
+
+captured_job = job_capture.last_job
+assert(captured_job[:delay_seconds] <= 0, "Past timestamp should result in immediate delivery")
+
+# Test current timestamp
+job_capture2 = JobCapture.new
+job_capture2.start_capturing
+
+job2 = ContinuableTestJob.new
+job2.sqs_send_message_parameters = {}
+
+current_timestamp = Time.current.to_f
+adapter3.enqueue_at(job2, current_timestamp)
+
+captured_job2 = job_capture2.last_job
+delay = captured_job2[:delay_seconds]
+assert(delay >= -1 && delay <= 1, "Current timestamp should have minimal delay")
+
+# Test future timestamp
+job_capture3 = JobCapture.new
+job_capture3.start_capturing
+
+job3 = ContinuableTestJob.new
+job3.sqs_send_message_parameters = {}
+
+future_timestamp = Time.current.to_f + 30
+adapter3.enqueue_at(job3, future_timestamp)
+
+captured_job3 = job_capture3.last_job
+delay3 = captured_job3[:delay_seconds]
+assert(delay3 > 0, "Future timestamp should have positive delay")
+assert(delay3 <= 30, "Delay should not exceed scheduled time")

data/spec/integration/retry_behavior/retry_behavior_spec.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# This spec tests retry behavior including ApproximateReceiveCount tracking
+# across message redeliveries.
+
+require 'concurrent'
+
+setup_sqs
+
+queue_name = DT.queue
+create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '2' })
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+
+# Atomic counter for fail tracking
+fail_counter = Concurrent::AtomicFixnum.new(2)
+
+worker_class = Class.new do
+  include Shoryuken::Worker
+
+  shoryuken_options auto_delete: false, batch: false
+
+  define_method(:perform) do |sqs_msg, body|
+    receive_count = sqs_msg.attributes['ApproximateReceiveCount'].to_i
+    DT[:receive_counts] << receive_count
+
+    if fail_counter.value > 0
+      fail_counter.decrement
+      raise "Simulated failure"
+    else
+      sqs_msg.delete
+    end
+  end
+end
+
+worker_class.get_shoryuken_options['queue'] = queue_name
+Shoryuken.register_worker(queue_name, worker_class)
+
+Shoryuken::Client.queues(queue_name).send_message(message_body: 'retry-count-test')
+
+poll_queues_until(timeout: 20) { DT[:receive_counts].size >= 3 }
+
+assert(DT[:receive_counts].size >= 3)
+assert_equal(DT[:receive_counts], DT[:receive_counts].sort, "Receive counts should be increasing")
+assert_equal(1, DT[:receive_counts].first)

data/spec/integration/spec_helper.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# Integration test spec helper
+# This file is auto-required by RSpec for integration tests
+
+require 'shoryuken'
+require_relative '../integrations_helper'

data/spec/integration/strict_priority_polling/strict_priority_polling_spec.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# This spec tests the StrictPriority polling strategy.
+# Higher priority queues are always processed before lower priority queues.
+
+setup_sqs
+
+queue_high = DT.queues[0]
+queue_low = DT.queues[1]
+
+[queue_high, queue_low].each { |q| create_test_queue(q) }
+
+# Configure StrictPriority polling strategy
+Shoryuken.options[:polling_strategy] = 'StrictPriority'
+
+Shoryuken.add_group('default', 1)
+# Higher weight = higher priority (queue_high appears 3 times, queue_low appears 1 time)
+Shoryuken.add_queue(queue_high, 3, 'default')
+Shoryuken.add_queue(queue_low, 1, 'default')
+
+worker_class = Class.new do
+  include Shoryuken::Worker
+
+  shoryuken_options auto_delete: true, batch: false
+
+  def perform(sqs_msg, body)
+    queue = sqs_msg.queue_url.split('/').last
+    DT[:processed_order] << { queue: queue, body: body, time: Time.now }
+  end
+end
+
+[queue_high, queue_low].each do |queue|
+  worker_class.get_shoryuken_options['queue'] = queue
+  Shoryuken.register_worker(queue, worker_class)
+end
+
+# Send messages to low priority queue first
+3.times { |i| Shoryuken::Client.queues(queue_low).send_message(message_body: "low-#{i}") }
+
+# Then send messages to high priority queue
+3.times { |i| Shoryuken::Client.queues(queue_high).send_message(message_body: "high-#{i}") }
+
+sleep 1
+
+poll_queues_until(timeout: 20) { DT[:processed_order].size >= 6 }
+
+assert_equal(6, DT[:processed_order].size)
+
+# With StrictPriority, high priority messages should generally be processed first
+high_messages = DT[:processed_order].select { |m| m[:queue] == queue_high }
+low_messages = DT[:processed_order].select { |m| m[:queue] == queue_low }
+
+assert_equal(3, high_messages.size, "All high priority messages should be processed")
+assert_equal(3, low_messages.size, "All low priority messages should be processed")
+
+# Verify both queues were processed
+queues_processed = DT[:processed_order].map { |m| m[:queue] }.uniq
+assert_equal(2, queues_processed.size, "Both queues should have messages processed")

data/spec/integration/visibility_timeout/visibility_timeout_spec.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# This spec tests visibility timeout management including manual visibility
+# extension during long processing.
+
+setup_sqs
+
+queue_name = DT.queue
+create_test_queue(queue_name, attributes: { 'VisibilityTimeout' => '5' })
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+
+worker_class = Class.new do
+  include Shoryuken::Worker
+
+  def perform(sqs_msg, body)
+    # Extend visibility before long processing
+    sqs_msg.change_visibility(visibility_timeout: 30)
+    DT[:visibility_extended] << true
+
+    sleep 2 # Simulate slow processing
+
+    DT[:messages] << body
+  end
+end
+
+worker_class.get_shoryuken_options['queue'] = queue_name
+worker_class.get_shoryuken_options['auto_delete'] = true
+worker_class.get_shoryuken_options['batch'] = false
+Shoryuken.register_worker(queue_name, worker_class)
+
+Shoryuken::Client.queues(queue_name).send_message(message_body: 'extend-test')
+
+poll_queues_until { DT[:messages].size >= 1 }
+
+assert_equal(1, DT[:messages].size)
+assert(DT[:visibility_extended].any?, "Expected visibility to be extended")

data/spec/integration/worker_enqueueing/worker_enqueueing_spec.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+# This spec tests the worker enqueueing methods:
+# - perform_async - enqueue a job for immediate processing
+# - perform_in - enqueue a job with a delay
+
+setup_sqs
+
+queue_name = DT.queue
+create_test_queue(queue_name)
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+
+# Worker for testing enqueueing methods
+enqueueing_worker = Class.new do
+  include Shoryuken::Worker
+
+  shoryuken_options auto_delete: true
+
+  def perform(sqs_msg, body)
+    DT[:processed_messages] << {
+      message_id: sqs_msg.message_id,
+      body: body,
+      processed_at: Time.now
+    }
+  end
+end
+
+enqueueing_worker.get_shoryuken_options['queue'] = queue_name
+Shoryuken.register_worker(queue_name, enqueueing_worker)
+
+# Test 1: perform_async - immediate enqueueing with string body
+enqueueing_worker.perform_async('async string message')
+
+# Test 2: perform_async with hash body
+enqueueing_worker.perform_async('action' => 'test', 'data' => [1, 2, 3])
+
+# Test 3: perform_in - delayed enqueueing (use short 1 second delay)
+enqueueing_worker.perform_in(1, 'delayed message')
+
+sleep 1
+
+# Poll for all 3 messages
+poll_queues_until(timeout: 15) { DT[:processed_messages].size >= 3 }
+
+assert_equal(3, DT[:processed_messages].size)
+
+# Verify string message was processed
+string_msg = DT[:processed_messages].find { |m| m[:body] == 'async string message' }
+assert(string_msg, 'String message should have been processed')
+
+# Verify hash message was processed (bodies might be stringified depending on serialization)
+hash_msg = DT[:processed_messages].find do |m|
+  m[:body].is_a?(Hash) || (m[:body].is_a?(String) && m[:body].include?('action'))
+end
+assert(hash_msg, 'Hash message should have been processed')
+
+# Verify delayed message was processed
+delayed_msg = DT[:processed_messages].find { |m| m[:body] == 'delayed message' }
+assert(delayed_msg, 'Delayed message should have been processed')

data/spec/integration/worker_groups/worker_groups_spec.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+# This spec tests multiple worker groups with different concurrency settings.
+# Each group can have its own queues and concurrency level.
+
+require 'concurrent'
+
+setup_sqs
+
+queue_group1 = DT.queues[0]
+queue_group2 = DT.queues[1]
+
+%w[queue_group1 queue_group2].each_with_index { |_, i| create_test_queue(DT.queues[i]) }
+
+# Configure two separate groups with different concurrency
+Shoryuken.add_group('group1', 3)  # 3 concurrent processors
+Shoryuken.add_group('group2', 1)  # 1 concurrent processor
+
+Shoryuken.add_queue(queue_group1, 1, 'group1')
+Shoryuken.add_queue(queue_group2, 1, 'group2')
+
+# Track concurrent processing per group
+group1_concurrent = Concurrent::AtomicFixnum.new(0)
+group1_max = Concurrent::AtomicFixnum.new(0)
+group2_concurrent = Concurrent::AtomicFixnum.new(0)
+group2_max = Concurrent::AtomicFixnum.new(0)
+
+worker_class = Class.new do
+  include Shoryuken::Worker
+
+  shoryuken_options auto_delete: true, batch: false
+
+  define_method(:perform) do |sqs_msg, body|
+    queue = sqs_msg.queue_url.split('/').last
+
+    if queue == queue_group1
+      group1_concurrent.increment
+      current = group1_concurrent.value
+      group1_max.update { |max| [max, current].max }
+      sleep 0.5  # Longer sleep to increase chance of concurrent execution
+      group1_concurrent.decrement
+    else
+      group2_concurrent.increment
+      current = group2_concurrent.value
+      group2_max.update { |max| [max, current].max }
+      sleep 0.3
+      group2_concurrent.decrement
+    end
+
+    DT[:processed] << { queue: queue, body: body }
+  end
+end
+
+[queue_group1, queue_group2].each do |queue|
+  worker_class.get_shoryuken_options['queue'] = queue
+  Shoryuken.register_worker(queue, worker_class)
+end
+
+# Send messages to both groups
+5.times { |i| Shoryuken::Client.queues(queue_group1).send_message(message_body: "group1-#{i}") }
+5.times { |i| Shoryuken::Client.queues(queue_group2).send_message(message_body: "group2-#{i}") }
+
+sleep 1
+
+poll_queues_until(timeout: 20) { DT[:processed].size >= 10 }
+
+assert_equal(10, DT[:processed].size)
+
+# Verify messages from both groups were processed
+group1_messages = DT[:processed].select { |m| m[:queue] == queue_group1 }
+group2_messages = DT[:processed].select { |m| m[:queue] == queue_group2 }
+
+assert_equal(5, group1_messages.size, 'All group1 messages should be processed')
+assert_equal(5, group2_messages.size, 'All group2 messages should be processed')
+
+# Verify concurrency was used - group1 with concurrency 3 should process concurrently
+# group2 with concurrency 1 should process sequentially (max = 1)
+assert(group1_max.value >= 1, "Group1 should have processed messages (max concurrent: #{group1_max.value})")
+assert_equal(1, group2_max.value, 'Group2 with concurrency 1 should process sequentially')

data/spec/integration/worker_lifecycle/worker_lifecycle_spec.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# This spec tests worker lifecycle including worker registration and discovery.
+
+setup_sqs
+
+queue_name = DT.queue
+create_test_queue(queue_name)
+Shoryuken.add_group('default', 1)
+Shoryuken.add_queue(queue_name, 1, 'default')
+
+worker_class = Class.new do
+  include Shoryuken::Worker
+
+  def perform(sqs_msg, body)
+    DT[:messages] << body
+  end
+end
+
+worker_class.get_shoryuken_options['queue'] = queue_name
+worker_class.get_shoryuken_options['auto_delete'] = true
+worker_class.get_shoryuken_options['batch'] = false
+Shoryuken.register_worker(queue_name, worker_class)
+
+registered = Shoryuken.worker_registry.workers(queue_name)
+assert_includes(registered, worker_class)
+
+Shoryuken::Client.queues(queue_name).send_message(message_body: 'lifecycle-test')
+
+poll_queues_until { DT[:messages].size >= 1 }
+
+assert_equal(1, DT[:messages].size)
+assert_equal('lifecycle-test', DT[:messages].first)

data/spec/integrations_helper.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+# Integration test helper for process-isolated testing
+
+# Enable Ruby warnings to catch deprecations and potential issues
+Warning[:performance] = true if RUBY_VERSION >= '3.3'
+Warning[:deprecated] = true
+$VERBOSE = true
+
+require 'warning'
+
+# Process warnings and raise on unexpected ones from our code
+Warning.process do |warning|
+  # Only check warnings from our code (not dependencies)
+  next unless warning.include?(Dir.pwd)
+
+  # Filter out warnings we don't care about in specs
+  next if warning.include?('_spec')
+
+  # We redefine methods to simulate various scenarios in tests
+  next if warning.include?('previous definition of')
+  next if warning.include?('method redefined')
+
+  # Ignore vendor directory
+  next if warning.include?('vendor/')
+
+  # Ignore bundle path
+  next if warning.include?('bundle/')
+
+  raise "Warning in your code: #{warning}"
+end
+
+require 'timeout'
+require 'json'
+require 'securerandom'
+require 'aws-sdk-sqs'
+require 'shoryuken'
+require 'singleton'
+
+# Thread-safe data collector for integration tests
+# Inspired by Karafka's DataCollector pattern
+# Usage: DT[:key] << value, DT[:key].size, DT.clear
+class DataCollector
+  include Singleton
+
+  MUTEX = Mutex.new
+  private_constant :MUTEX
+
+  attr_reader :queues, :data
+
+  class << self
+    def queue
+      instance.queue
+    end
+
+    def queues
+      instance.queues
+    end
+
+    def data
+      instance.data
+    end
+
+    def [](key)
+      MUTEX.synchronize { data[key] }
+    end
+
+    def []=(key, value)
+      MUTEX.synchronize { data[key] = value }
+    end
+
+    def uuids(amount)
+      Array.new(amount) { uuid }
+    end
+
+    def uuid
+      "it-#{SecureRandom.uuid[0, 8]}"
+    end
+
+    def clear
+      MUTEX.synchronize { instance.clear }
+    end
+
+    def key?(key)
+      instance.data.key?(key)
+    end
+  end
+
+  def initialize
+    @mutex = Mutex.new
+    @queues = Array.new(100) { "it-#{SecureRandom.hex(6)}" }
+    @data = Hash.new do |hash, key|
+      @mutex.synchronize do
+        break hash[key] if hash.key?(key)
+
+        hash[key] = []
+      end
+    end
+  end
+
+  def queue
+    queues.first
+  end
+
+  def clear
+    @mutex.synchronize do
+      @queues.clear
+      @queues.concat(Array.new(100) { "it-#{SecureRandom.hex(6)}" })
+      @data.clear
+    end
+  end
+end
+
+# Short alias for DataCollector
+DT = DataCollector
+
+module IntegrationsHelper
+  class TestFailure < StandardError; end
+
+  # Assertions
+  def assert(condition, message = "Assertion failed")
+    raise TestFailure, message unless condition
+  end
+
+  def assert_equal(expected, actual, message = nil)
+    message ||= "Expected #{expected.inspect}, got #{actual.inspect}"
+    assert(expected == actual, message)
+  end
+
+  def assert_includes(collection, item, message = nil)
+    message ||= "Expected #{collection.inspect} to include #{item.inspect}"
+    assert(collection.include?(item), message)
+  end
+
+  def refute(condition, message = "Refutation failed")
+    assert(!condition, message)
+  end
+
+  # Configure ActiveJob with Shoryuken adapter
+  def setup_active_job
+    require 'active_job'
+    require 'active_job/queue_adapters/shoryuken_adapter'
+    require 'active_job/extensions'
+
+    ActiveJob::Base.queue_adapter = :shoryuken
+  end
+
+  # Configure Shoryuken to use ElasticMQ for real SQS integration tests
+  def setup_sqs
+    Aws.config[:stub_responses] = false
+
+    sqs_client = Aws::SQS::Client.new(
+      region: 'us-east-1',
+      endpoint: 'http://localhost:9324',
+      access_key_id: 'fake',
+      secret_access_key: 'fake'
+    )
+
+    Shoryuken.options[:concurrency] = 25
+    Shoryuken.options[:delay] = 0
+    Shoryuken.options[:timeout] = 8
+
+    executor = Concurrent::CachedThreadPool.new(auto_terminate: true)
+    Shoryuken.define_singleton_method(:launcher_executor) { executor }
+
+    Shoryuken.configure_client { |config| config.sqs_client = sqs_client }
+    Shoryuken.configure_server { |config| config.sqs_client = sqs_client }
+  end
+
+  # Queue helpers
+  def create_test_queue(queue_name, attributes: {})
+    Shoryuken::Client.sqs.create_queue(queue_name: queue_name, attributes: attributes)
+  end
+
+  def delete_test_queue(queue_name)
+    queue_url = Shoryuken::Client.sqs.get_queue_url(queue_name: queue_name).queue_url
+    Shoryuken::Client.sqs.delete_queue(queue_url: queue_url)
+  rescue Aws::SQS::Errors::NonExistentQueue
+  end
+
+  def create_fifo_queue(queue_name)
+    create_test_queue(queue_name, attributes: {
+      'FifoQueue' => 'true',
+      'ContentBasedDeduplication' => 'true'
+    })
+  end
+
+  # Poll until condition met
+  def poll_queues_until(timeout: 15)
+    launcher = Shoryuken::Launcher.new
+    launcher.start
+    Timeout.timeout(timeout) { sleep 0.5 until yield }
+  ensure
+    launcher.stop
+  end
+
+  # Simple mock object
+  def double(_name = nil)
+    Object.new
+  end
+
+  # Job capture for ActiveJob tests
+  class JobCapture
+    attr_reader :jobs
+
+    def initialize
+      @jobs = []
+    end
+
+    def start_capturing
+      @jobs.clear
+      capture = self
+
+      queue_mock = Object.new
+      queue_mock.define_singleton_method(:fifo?) { false }
+      queue_mock.define_singleton_method(:send_message) do |params|
+        capture.jobs << {
+          queue: params[:queue_name] || :default,
+          message_body: params[:message_body],
+          delay_seconds: params[:delay_seconds],
+          message_attributes: params[:message_attributes]
+        }
+      end
+
+      Shoryuken::Client.define_singleton_method(:queues) do |queue_name = nil|
+        queue_mock.define_singleton_method(:name) { queue_name } if queue_name
+        queue_name ? queue_mock : { default: queue_mock }
+      end
+
+      Shoryuken.define_singleton_method(:register_worker) { |*| nil }
+    end
+
+    def last_job
+      @jobs.last
+    end
+
+    def job_count
+      @jobs.size
+    end
+  end
+end
+
+include IntegrationsHelper