shoryuken 7.0.0.alpha2 → 7.0.0

data/bin/cli/sqs.rb

@@ -1,19 +1,28 @@
+# frozen_string_literal: true
+
 require 'date'
 
 # rubocop:disable Metrics/BlockLength
 module Shoryuken
   module CLI
+    # SQS command line interface for queue management operations.
+    # Provides commands for listing, creating, deleting, moving, and dumping queue messages.
     class SQS < Base
-      # See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
+      # Maximum batch size in bytes for SQS batch operations
+      # @see https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
       MAX_BATCH_SIZE = 1024 * 1024
 
       namespace :sqs
       class_option :endpoint, aliases: '-e', type: :string, default: ENV['SHORYUKEN_SQS_ENDPOINT'], desc: 'Endpoint URL'
 
       no_commands do
+        # Normalizes a dump message for requeuing
+        #
+        # @param message [Hash] the dumped message hash
+        # @return [Hash] the normalized message
         def normalize_dump_message(message)
           # symbolize_keys is needed for keeping it compatible with `requeue`
-          attributes = message[:attributes].symbolize_keys
+          attributes = message[:attributes].transform_keys(&:to_sym)
 
           # See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_MessageAttributeValue.html
           # The `string_list_values` and `binary_list_values` are not implemented. Reserved for future use.
@@ -30,6 +39,9 @@ module Shoryuken
           }
         end
 
+        # Returns the SQS client options based on CLI options
+        #
+        # @return [Hash] the client options hash
         def client_options
           endpoint = options[:endpoint]
           {}.tap do |hash|
@@ -37,16 +49,28 @@ module Shoryuken
           end
         end
 
+        # Returns the SQS client instance
+        #
+        # @return [Aws::SQS::Client] the SQS client
         def sqs
           @_sqs ||= Aws::SQS::Client.new(client_options)
         end
 
+        # Finds the URL for a queue by name
+        #
+        # @param queue_name [String] the queue name
+        # @return [String] the queue URL
         def find_queue_url(queue_name)
           sqs.get_queue_url(queue_name: queue_name).queue_url
         rescue Aws::SQS::Errors::NonExistentQueue
           fail_task "The specified queue #{queue_name} does not exist"
         end
 
+        # Batch deletes messages from a queue
+        #
+        # @param url [String] the queue URL
+        # @param messages [Array<Aws::SQS::Types::Message>] the messages to delete
+        # @return [void]
         def batch_delete(url, messages)
           messages.to_a.flatten.each_slice(10) do |batch|
             sqs.delete_message_batch(
@@ -61,11 +85,23 @@ module Shoryuken
           end
         end
 
+        # Batch sends messages to a queue
+        #
+        # @param url [String] the queue URL
+        # @param messages [Array<Hash>] the messages to send
+        # @param max_batch_size [Integer] maximum messages per batch
+        # @return [void]
         def batch_send(url, messages, max_batch_size = 10)
           messages = messages.to_a.flatten.map(&method(:normalize_dump_message))
           batch_send_normalized_messages url, messages, max_batch_size
         end
 
+        # Batch sends normalized messages to a queue
+        #
+        # @param url [String] the queue URL
+        # @param messages [Array<Hash>] the normalized messages
+        # @param max_batch_size [Integer] maximum messages per batch
+        # @return [void]
         def batch_send_normalized_messages(url, messages, max_batch_size)
           # Repeatedly take the longest prefix of messages such that
           # 1. The number of messages is less than or equal to max_batch_size
@@ -90,10 +126,18 @@ module Shoryuken
           end
         end
 
+        # Calculates the total payload size of a batch
+        #
+        # @param messages [Array<Hash>] the messages
+        # @return [Integer] total size in bytes
         def batch_payload_size(messages)
           messages.sum(&method(:message_size))
         end
 
+        # Calculates the size of a single message
+        #
+        # @param message [Hash] the message
+        # @return [Integer] size in bytes
         def message_size(message)
           attribute_size = (message[:message_attributes] || []).sum do |name, value|
             name_size = name.to_s.bytesize
@@ -111,6 +155,12 @@ module Shoryuken
           attribute_size + body_size
         end
 
+        # Receives all messages from a queue up to a limit
+        #
+        # @param url [String] the queue URL
+        # @param limit [Integer, Float] maximum messages to receive
+        # @yield [Aws::SQS::Types::Message] yields each received message
+        # @return [Integer] the number of messages received
         def find_all(url, limit, &block)
           count = 0
           batch_size = limit > 10 ? 10 : limit
@@ -137,6 +187,10 @@ module Shoryuken
           count
         end
 
+        # Lists queues and prints their attributes as a table
+        #
+        # @param urls [Array<String>] the queue URLs
+        # @return [void]
         def list_and_print_queues(urls)
           attrs = %w[QueueArn ApproximateNumberOfMessages ApproximateNumberOfMessagesNotVisible LastModifiedTimestamp]
 
@@ -154,6 +208,11 @@ module Shoryuken
           print_table(entries)
         end
 
+        # Generates the dump file path for a queue
+        #
+        # @param path [String] the directory path
+        # @param queue_name [String] the queue name
+        # @return [String] the full file path
         def dump_file(path, queue_name)
           File.join(path, "#{queue_name}-#{Date.today}.jsonl")
         end

data/bin/integrations

@@ -0,0 +1,275 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Shoryuken integration test runner
+#
+# Usage:
+#   bin/integrations                    # Run all integration tests
+#   bin/integrations fifo               # Run tests with 'fifo' in path
+#   bin/integrations rails/rails_72     # Run Rails 7.2 tests
+#   bin/integrations batch retry        # Run tests matching 'batch' OR 'retry'
+#   bin/integrations -v fifo            # Run with verbose output
+
+require 'bundler'
+require 'fileutils'
+require 'timeout'
+
+TIMEOUT = 300 # 5 minutes per scenario
+SPEC_DIR = File.expand_path('../spec/integration', __dir__)
+ROOT_DIR = File.expand_path('..', __dir__)
+
+class IntegrationRunner
+  def initialize(args)
+    @verbose = args.delete('-v') || args.delete('--verbose')
+    @filters = args.reject { |a| a.start_with?('-') }
+  end
+
+  def run
+    specs = find_specs
+    specs = filter_specs(specs) if @filters.any?
+
+    if specs.empty?
+      puts 'No specs found matching filters'
+      exit 1
+    end
+
+    puts "Running #{specs.size} integration specs..."
+    puts
+
+    results = run_specs(specs)
+    report_results(results)
+  end
+
+  private
+
+  def find_specs
+    Dir.glob(File.join(SPEC_DIR, '**/*_spec.rb')).reject do |path|
+      # Exclude vendor and .bundle directories
+      path.include?('/vendor/') || path.include?('/.bundle/')
+    end.map do |path|
+      relative_path = path.sub("#{SPEC_DIR}/", '')
+      dir = File.dirname(path)
+      gemfile = File.exist?(File.join(dir, 'Gemfile')) ? File.join(dir, 'Gemfile') : File.join(ROOT_DIR, 'Gemfile')
+
+      {
+        name: relative_path.sub('_spec.rb', '').gsub('/', ' / '),
+        path: path,
+        relative_path: relative_path,
+        directory: dir,
+        gemfile: gemfile
+      }
+    end.sort_by { |s| s[:relative_path] }
+  end
+
+  def filter_specs(specs)
+    specs.select do |spec|
+      @filters.any? { |filter| spec[:relative_path].include?(filter) }
+    end
+  end
+
+  def run_specs(specs)
+    results = []
+
+    specs.each do |spec|
+      result = run_spec(spec)
+      results << result
+
+      if result[:skipped]
+        print 'S'
+      elsif result[:success]
+        print '.'
+      else
+        print 'F'
+      end
+      $stdout.flush
+    end
+
+    puts
+    results
+  end
+
+  def run_spec(spec)
+    # Start with a clean bundler environment to prevent pollution between tests
+    env = {
+      'BUNDLE_GEMFILE' => spec[:gemfile],
+      'RAILS_ENV' => 'test',
+      'RUBYOPT' => nil  # Clear any -rbundler/setup from CI or previous tests
+    }
+
+    # Install dependencies if using a local Gemfile
+    uses_local_gemfile = spec[:gemfile] != File.join(ROOT_DIR, 'Gemfile')
+    if uses_local_gemfile
+      install_result = install_bundle(spec, env)
+      unless install_result[:success]
+        # Skip test if bundle install fails (e.g., gems not available in CI)
+        return {
+          spec: spec,
+          success: true,
+          skipped: true,
+          skip_reason: 'Bundle install failed (dependencies not available)',
+          output: install_result[:output]
+        }
+      end
+
+      # Use isolated bundle config to match install_bundle
+      bundle_path = File.join(spec[:directory], 'vendor', 'bundle')
+      bundle_config = File.join(spec[:directory], '.bundle')
+      env['BUNDLE_PATH'] = bundle_path
+      env['BUNDLE_FROZEN'] = 'false'
+      env['BUNDLE_APP_CONFIG'] = bundle_config
+    end
+
+    # Run the spec
+    # For local gemfiles, use standalone bundle setup which doesn't need bundler at runtime
+    # This avoids issues with bundle exec inheriting the wrong config
+    cmd = if uses_local_gemfile
+            standalone_setup = File.join(bundle_path, 'bundler', 'setup.rb')
+            ['ruby', "-r#{standalone_setup}", File.join(ROOT_DIR, 'bin/scenario'), spec[:path]]
+          else
+            ['bundle', 'exec', 'ruby', File.join(ROOT_DIR, 'bin/scenario'), spec[:path]]
+          end
+
+    output = []
+    start_time = Time.now
+
+    begin
+      Timeout.timeout(TIMEOUT) do
+        # Use unbundled env to prevent pollution from previous test runs
+        # This is especially important after Rails integration tests that use
+        # bundle install --standalone with different gem versions
+        Bundler.with_unbundled_env do
+          IO.popen(env, cmd, chdir: spec[:directory], err: [:child, :out]) do |io|
+            io.each_line { |line| output << line }
+          end
+        end
+      end
+
+      {
+        spec: spec,
+        success: $?.success?,
+        exit_code: $?.exitstatus,
+        duration: Time.now - start_time,
+        output: output.join
+      }
+    rescue Timeout::Error
+      {
+        spec: spec,
+        success: false,
+        exit_code: -1,
+        error: 'Timeout',
+        duration: Time.now - start_time,
+        output: output.join
+      }
+    end
+  end
+
+  def install_bundle(spec, env)
+    return { success: true } if @bundle_installed&.include?(spec[:gemfile])
+
+    output = []
+
+    # Create isolated bundle environment to avoid CI cache interference
+    # Use a unique path per Gemfile to avoid conflicts
+    bundle_path = File.join(spec[:directory], 'vendor', 'bundle')
+    bundle_config = File.join(spec[:directory], '.bundle')
+
+    # Create local .bundle/config to override project-level config
+    FileUtils.mkdir_p(bundle_config)
+    File.write(File.join(bundle_config, 'config'), <<~CONFIG)
+      ---
+      BUNDLE_PATH: "#{bundle_path}"
+      BUNDLE_FROZEN: "false"
+    CONFIG
+
+    clean_env = env.merge(
+      'BUNDLE_PATH' => bundle_path,
+      'BUNDLE_FROZEN' => 'false',
+      'BUNDLE_DEPLOYMENT' => nil,
+      'BUNDLE_WITHOUT' => nil,
+      'BUNDLE_CACHE_PATH' => nil,
+      'BUNDLE_BIN' => nil,
+      'BUNDLE_APP_CONFIG' => bundle_config,
+      'RUBYOPT' => nil  # Clear any -rbundler/setup from CI
+    )
+
+    # Use --standalone to generate a setup.rb that doesn't need bundler at runtime
+    # Run in a completely unbundled environment using Bundler API
+    cmd_script = <<~RUBY
+      require 'bundler'
+      Bundler.with_unbundled_env do
+        system({'BUNDLE_GEMFILE' => '#{spec[:gemfile]}', 'BUNDLE_PATH' => '#{bundle_path}', 'BUNDLE_FROZEN' => 'false', 'BUNDLE_APP_CONFIG' => '#{bundle_config}'}, 'bundle', 'install', '--standalone')
+      end
+      exit($?.success? ? 0 : 1)
+    RUBY
+
+    IO.popen(['ruby', '-e', cmd_script], chdir: spec[:directory], err: [:child, :out]) do |io|
+      io.each_line { |line| output << line }
+    end
+
+    @bundle_installed ||= []
+    @bundle_installed << spec[:gemfile] if $?.success?
+
+    {
+      spec: spec,
+      success: $?.success?,
+      output: output.join,
+      error: $?.success? ? nil : 'Bundle install failed',
+      bundle_config: bundle_config
+    }
+  end
+
+  def report_results(results)
+    skipped = results.select { |r| r[:skipped] }
+    failed = results.reject { |r| r[:success] || r[:skipped] }
+    passed = results.count { |r| r[:success] && !r[:skipped] }
+    total = results.size
+
+    puts
+    summary = "#{passed}/#{total} passed"
+    summary += ", #{skipped.size} skipped" if skipped.any?
+    puts summary
+
+    if skipped.any?
+      puts
+      puts 'Skipped:'
+      puts
+      skipped.each do |result|
+        puts "  - #{result[:spec][:name]}"
+        puts "    Reason: #{result[:skip_reason]}" if result[:skip_reason]
+        if result[:output] && !result[:output].strip.empty?
+          lines = result[:output].lines.last(15)
+          lines.each { |line| puts "    #{line}" }
+        end
+      end
+    end
+
+    if failed.any?
+      puts
+      puts 'Failures:'
+      puts
+
+      failed.each_with_index do |result, idx|
+        puts "  #{idx + 1}) #{result[:spec][:name]}"
+        if result[:error]
+          puts "     Error: #{result[:error]}"
+        end
+        if result[:output] && !result[:output].strip.empty?
+          # Show last 30 lines of output for context
+          lines = result[:output].lines
+          if lines.size > 30
+            puts "     ... (#{lines.size - 30} lines truncated)"
+            lines = lines.last(30)
+          end
+          lines.each { |line| puts "     #{line}" }
+        end
+        puts
+      end
+    end
+
+    exit(failed.empty? ? 0 : 1)
+  end
+end
+
+if __FILE__ == $0
+  IntegrationRunner.new(ARGV.dup).run
+end

data/bin/scenario

@@ -0,0 +1,154 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Individual scenario runner for integration testing
+# This script runs a single integration test file in complete isolation
+
+require 'bundler/setup'
+
+# Exit codes
+EXIT_SUCCESS = 0
+EXIT_FAILURE = 1
+EXIT_TIMEOUT = 2
+EXIT_SETUP_ERROR = 3
+
+class ScenarioRunner
+  attr_reader :test_file
+
+  def initialize(test_file)
+    @test_file = test_file
+    @exit_code = EXIT_SUCCESS
+  end
+
+  def run
+    puts "Running: #{File.basename(test_file)}" if ENV['VERBOSE']
+
+    # Set up the scenario-specific environment
+    setup_scenario
+
+    # Load and run the test file
+    load_and_run_test
+
+    exit EXIT_SUCCESS
+  rescue => e
+    puts "FAILED: #{File.basename(test_file)} - #{e.message}" if ENV['VERBOSE']
+    puts e.backtrace.first(5).join("\n") if ENV['VERBOSE']
+    exit EXIT_FAILURE
+  end
+
+  private
+
+  def setup_scenario
+    # Each test handles its own specific requirements
+    require 'bundler/setup'
+
+    puts "Setting up isolated test environment" if ENV['VERBOSE']
+  end
+
+
+  def load_and_run_test
+    # Test file might be relative to current directory
+    if File.exist?(test_file)
+      absolute_test_path = File.expand_path(test_file)
+    else
+      # Fallback to project root resolution
+      project_root = File.expand_path('..', __dir__)
+      absolute_test_path = File.join(project_root, test_file)
+    end
+
+    puts "Current directory: #{Dir.pwd}" if ENV['VERBOSE']
+    puts "Loading test file: #{absolute_test_path}" if ENV['VERBOSE']
+
+    unless File.exist?(absolute_test_path)
+      raise "Test file not found: #{absolute_test_path}"
+    end
+
+    # Check if this is an RSpec file (contains RSpec.describe or describe)
+    file_content = File.read(absolute_test_path, encoding: 'UTF-8')
+    if file_content.match?(/\b(?:RSpec\.describe|describe)\b/)
+      puts "Running as RSpec test" if ENV['VERBOSE']
+      run_rspec_test(absolute_test_path)
+    else
+      puts "Running as plain Ruby test" if ENV['VERBOSE']
+      # Load integrations_helper for plain Ruby integration tests
+      if absolute_test_path.include?('spec/integration')
+        project_root = File.expand_path('..', __dir__)
+        integrations_helper = File.join(project_root, 'spec', 'integrations_helper.rb')
+        require integrations_helper
+      end
+      # Load as plain Ruby test
+      load absolute_test_path
+    end
+  end
+
+  def run_rspec_test(test_file_path)
+    # Change to project root for RSpec to find spec_helper
+    project_root = File.expand_path('..', __dir__)
+    Dir.chdir(project_root) do
+      # Disable SimpleCov for integration tests to avoid coverage failures
+      ENV['SIMPLECOV_DISABLED'] = 'true'
+
+      # Make the test file path relative to project root for RSpec
+      relative_test_path = test_file_path.sub("#{project_root}/", '')
+
+      puts "Running RSpec with file: #{relative_test_path}" if ENV['VERBOSE']
+      puts "Working directory: #{Dir.pwd}" if ENV['VERBOSE']
+
+      # Check if this test requires Rails but Rails is not available
+      if requires_rails?(test_file_path) && !rails_available?
+        puts "Skipping #{File.basename(test_file_path)} - Rails not available"
+        return
+      end
+
+      # Run RSpec with the specific test file
+      require 'rspec/core'
+
+      # Load integration spec_helper for integration tests
+      if relative_test_path.include?('spec/integration/')
+        require_relative '../spec/integration/spec_helper'
+      end
+
+      result = RSpec::Core::Runner.run([relative_test_path], $stderr, $stdout)
+
+      if result != 0
+        raise "RSpec failed with exit code #{result}"
+      end
+    ensure
+      # Clean up environment
+      ENV.delete('SIMPLECOV_DISABLED')
+    end
+  end
+
+  def requires_rails?(test_file_path)
+    # Check if the test file mentions Rails dependencies
+    content = File.read(test_file_path)
+    content.match?(/require.*rails|Rails::|ActiveJob::|ActionController::/)
+  end
+
+  def rails_available?
+    begin
+      require 'rails'
+      true
+    rescue LoadError
+      false
+    end
+  end
+end
+
+# Validate arguments
+if ARGV.empty?
+  puts "Usage: bin/scenario <test_file>"
+  puts "Example: bin/scenario spec/integration/rails_integration_spec.rb"
+  exit EXIT_SETUP_ERROR
+end
+
+test_file = ARGV[0]
+
+unless File.exist?(test_file)
+  puts "Test file not found: #{test_file}"
+  exit EXIT_SETUP_ERROR
+end
+
+
+# Run the scenario
+ScenarioRunner.new(test_file).run

data/bin/shoryuken

@@ -29,7 +29,7 @@ module Shoryuken
       method_option :delay,       aliases: '-D', type: :numeric,
                                   desc: 'Number of seconds to pause fetching from an empty queue'
       def start
-        opts = options.to_h.symbolize_keys
+        opts = options.to_h.transform_keys(&:to_sym)
 
         say '[DEPRECATED] Please use --config instead of --config-file', :yellow if opts[:config_file]
 

data/lib/{shoryuken/extensions/active_job_extensions.rb → active_job/extensions.rb}

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Shoryuken
-  module ActiveJobExtensions
+  module ActiveJob
     # Adds an accessor for SQS SendMessage parameters on ActiveJob jobs
     # (instances of ActiveJob::Base). Shoryuken ActiveJob queue adapters use
     # these parameters when enqueueing jobs; other adapters can ignore them.
@@ -9,6 +9,7 @@ module Shoryuken
       extend ActiveSupport::Concern
 
       included do
+        # @return [Hash] the SQS send message parameters
         attr_accessor :sqs_send_message_parameters
       end
     end
@@ -17,12 +18,22 @@ module Shoryuken
     # to the empty hash, and populates it whenever `#enqueue` is called, such
     # as when using ActiveJob::Base.set.
     module SQSSendMessageParametersSupport
+      # Initializes a new ActiveJob instance with empty SQS parameters
+      #
+      # @param arguments [Array] the job arguments
       def initialize(*arguments)
         super(*arguments)
         self.sqs_send_message_parameters = {}
       end
-      ruby2_keywords(:initialize) if respond_to?(:ruby2_keywords, true)
 
+      # Enqueues the job with optional SQS-specific parameters
+      #
+      # @param options [Hash] enqueue options
+      # @option options [Hash] :message_attributes custom SQS message attributes
+      # @option options [Hash] :message_system_attributes system attributes
+      # @option options [String] :message_deduplication_id FIFO deduplication ID
+      # @option options [String] :message_group_id FIFO message group ID
+      # @return [Object] the enqueue result
       def enqueue(options = {})
         sqs_options = options.extract! :message_attributes,
                                        :message_system_attributes,
@@ -36,5 +47,5 @@ module Shoryuken
   end
 end
 
-ActiveJob::Base.include Shoryuken::ActiveJobExtensions::SQSSendMessageParametersAccessor
-ActiveJob::Base.prepend Shoryuken::ActiveJobExtensions::SQSSendMessageParametersSupport
+ActiveJob::Base.include Shoryuken::ActiveJob::SQSSendMessageParametersAccessor
+ActiveJob::Base.prepend Shoryuken::ActiveJob::SQSSendMessageParametersSupport