shoryuken 6.2.1 → 7.0.2

data/lib/shoryuken/environment_loader.rb

@@ -1,28 +1,61 @@
+# frozen_string_literal: true
+
 module Shoryuken
+  # Loads and configures the Shoryuken environment from configuration files
+  # and command line options. Handles Rails integration and queue setup.
   class EnvironmentLoader
+    # @return [Hash] the configuration options
     attr_reader :options
 
+    # Sets up a new EnvironmentLoader with the given options
+    #
+    # @param options [Hash] configuration options
+    # @option options [String] :config_file path to the configuration file
+    # @option options [Boolean] :rails whether to initialize Rails
+    # @option options [String] :logfile path to the log file
+    # @option options [Boolean] :verbose whether to enable verbose logging
+    # @option options [String] :require path to require workers from
+    # @option options [Integer] :concurrency number of concurrent workers
+    # @return [Shoryuken::EnvironmentLoader] the configured instance
     def self.setup_options(options)
       instance = new(options)
       instance.setup_options
       instance
     end
 
+    # Loads the environment for Rails console usage
+    #
+    # @return [void]
     def self.load_for_rails_console
       instance = setup_options(config_file: (Rails.root + 'config' + 'shoryuken.yml'))
       instance.load
     end
 
+    # Initializes a new EnvironmentLoader with the given options
+    #
+    # @param options [Hash] configuration options
+    # @option options [String] :config_file path to the configuration file
+    # @option options [Boolean] :rails whether to initialize Rails
+    # @option options [String] :logfile path to the log file
+    # @option options [Boolean] :verbose whether to enable verbose logging
+    # @option options [String] :require path to require workers from
+    # @option options [Integer] :concurrency number of concurrent workers
     def initialize(options)
       @options = options
     end
 
+    # Sets up configuration options from file and initializes components
+    #
+    # @return [void]
     def setup_options
       initialize_rails if load_rails?
       initialize_options
       initialize_logger
     end
 
+    # Loads the environment including queues and workers
+    #
+    # @return [void]
     def load
       prefix_active_job_queue_names
       parse_queues
@@ -33,28 +66,40 @@ module Shoryuken
 
     private
 
+    # Merges configuration file options with runtime options
+    #
+    # @return [void]
     def initialize_options
       Shoryuken.options.merge!(config_file_options)
       Shoryuken.options.merge!(options)
     end
 
+    # Reads and parses the configuration file
+    #
+    # @return [Hash] the parsed configuration options
     def config_file_options
       return {} unless (path = options[:config_file])
 
-      fail ArgumentError, "The supplied config file #{path} does not exist" unless File.exist?(path)
+      raise Errors::InvalidConfigurationError, "The supplied config file #{path} does not exist" unless File.exist?(path)
 
       if (result = YAML.load(ERB.new(IO.read(path)).result))
-        result.deep_symbolize_keys
+        Shoryuken::Helpers::HashUtils.deep_symbolize_keys(result)
       else
         {}
       end
     end
 
+    # Initializes the logger with file output and verbosity settings
+    #
+    # @return [void]
     def initialize_logger
       Shoryuken::Logging.initialize_logger(Shoryuken.options[:logfile]) if Shoryuken.options[:logfile]
       Shoryuken.logger.level = Logger::DEBUG if Shoryuken.options[:verbose]
     end
 
+    # Initializes the Rails environment
+    #
+    # @return [void]
     def initialize_rails
       # Adapted from: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/cli.rb
 
@@ -78,18 +123,26 @@ module Shoryuken
           end
         end
         if Shoryuken.active_job?
-          require 'shoryuken/extensions/active_job_extensions'
-          require 'shoryuken/extensions/active_job_adapter'
-          require 'shoryuken/extensions/active_job_concurrent_send_adapter'
+          require 'active_job/extensions'
+          require 'active_job/queue_adapters/shoryuken_adapter'
+          require 'active_job/queue_adapters/shoryuken_concurrent_send_adapter'
         end
         require File.expand_path('config/environment.rb')
       end
     end
 
+    # Checks if Rails should be loaded
+    #
+    # @return [Boolean] true if Rails should be initialized
     def load_rails?
       options[:rails]
     end
 
+    # Prefixes a queue name with ActiveJob queue name prefix
+    #
+    # @param queue_name [String] the queue name to prefix
+    # @param weight [Integer] the queue weight
+    # @return [Array<String, Integer>] the prefixed queue name and weight
     def prefix_active_job_queue_name(queue_name, weight)
       return [queue_name, weight] if queue_name.start_with?('https://', 'arn:')
 
@@ -102,6 +155,9 @@ module Shoryuken
       [prefixed_queue_name, weight]
     end
 
+    # Prefixes all queue names with ActiveJob prefix if enabled
+    #
+    # @return [void]
     def prefix_active_job_queue_names
       return unless Shoryuken.active_job?
       return unless Shoryuken.active_job_queue_name_prefixing?
@@ -121,10 +177,19 @@ module Shoryuken
       end
     end
 
+    # Parses a single queue and adds it to a group
+    #
+    # @param queue [String] the queue name
+    # @param weight [Integer] the queue weight
+    # @param group [String] the group name
+    # @return [void]
     def parse_queue(queue, weight, group)
       Shoryuken.add_queue(queue, [weight.to_i, 1].max, group)
     end
 
+    # Parses all queues from configuration and adds them to groups
+    #
+    # @return [void]
     def parse_queues
       if Shoryuken.options[:queues].to_a.any?
         Shoryuken.add_group('default', Shoryuken.options[:concurrency])
@@ -135,7 +200,7 @@ module Shoryuken
       end
 
       Shoryuken.options[:groups].to_a.each do |group, options|
-        Shoryuken.add_group(group, options[:concurrency], delay: options[:delay])
+        Shoryuken.add_group(group, options[:concurrency], delay: options[:delay], polling_strategy: options[:polling_strategy])
 
         options[:queues].to_a.each do |queue, weight|
           parse_queue(queue, weight, group)
@@ -143,6 +208,9 @@ module Shoryuken
       end
     end
 
+    # Requires worker files from the configured path
+    #
+    # @return [void]
     def require_workers
       required = Shoryuken.options[:require]
 
@@ -155,17 +223,19 @@ module Shoryuken
       end
     end
 
+    # Validates that all configured queues exist in SQS
+    #
+    # @return [void]
+    # @raise [ArgumentError] if any queues do not exist
     def validate_queues
       return Shoryuken.logger.warn { 'No queues supplied' } if Shoryuken.ungrouped_queues.empty?
 
       non_existent_queues = []
 
       Shoryuken.ungrouped_queues.uniq.each do |queue|
-        begin
-          Shoryuken::Client.queues(queue)
-        rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue
-          non_existent_queues << queue
-        end
+        Shoryuken::Client.queues(queue)
+      rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue
+        non_existent_queues << queue
       end
 
       return if non_existent_queues.none?
@@ -178,12 +248,12 @@ module Shoryuken
         It's also possible that you don't have permission to access the specified queues.
       MSG
 
-      fail(
-        ArgumentError,
-        error_msg
-      )
+      raise Errors::QueueNotFoundError, error_msg
     end
 
+    # Validates that all queues have registered workers
+    #
+    # @return [void]
     def validate_workers
       return if Shoryuken.active_job?
 

data/lib/shoryuken/errors.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  # Namespace for all Shoryuken-specific errors.
+  # These provide more meaningful error types than generic Ruby exceptions,
+  # making it easier to rescue and handle specific failure cases.
+  module Errors
+    # Base class for all Shoryuken errors
+    BaseError = Class.new(StandardError)
+
+    # Raised when there is a configuration validation failure
+    InvalidConfigurationError = Class.new(BaseError)
+
+    # Raised when a specified SQS queue does not exist or cannot be accessed
+    QueueNotFoundError = Class.new(BaseError)
+
+    # Raised when worker registration fails due to conflicts
+    # (e.g., registering multiple workers for a batch queue)
+    InvalidWorkerRegistrationError = Class.new(BaseError)
+
+    # Raised when an invalid polling strategy is specified
+    InvalidPollingStrategyError = Class.new(BaseError)
+
+    # Raised when an invalid lifecycle event name is used
+    InvalidEventError = Class.new(BaseError)
+
+    # Raised when a delay exceeds the maximum allowed by SQS (15 minutes)
+    InvalidDelayError = Class.new(BaseError)
+
+    # Raised when a delay is used with a FIFO queue
+    FifoDelayNotSupportedError = Class.new(BaseError)
+
+    # Raised when an ARN format is invalid
+    InvalidArnError = Class.new(BaseError)
+  end
+end

data/lib/shoryuken/fetcher.rb

@@ -1,13 +1,26 @@
+# frozen_string_literal: true
+
 module Shoryuken
+  # Fetches messages from SQS queues.
+  # Handles message retrieval with automatic retry on connectivity errors.
   class Fetcher
     include Util
 
+    # Maximum number of messages that can be fetched in a single SQS request
     FETCH_LIMIT = 10
 
+    # Initializes a new Fetcher for a processing group
+    #
+    # @param group [String] the processing group name
     def initialize(group)
       @group = group
     end
 
+    # Fetches messages from a queue with automatic retry
+    #
+    # @param queue [Shoryuken::Polling::QueueConfiguration] the queue configuration
+    # @param limit [Integer] the maximum number of messages to fetch
+    # @return [Array<Aws::SQS::Types::Message>] the fetched messages
     def fetch(queue, limit)
       fetch_with_auto_retry(3) do
         started_at = Time.now
@@ -25,18 +38,23 @@ module Shoryuken
 
     private
 
+    # Fetches with automatic retry on errors
+    #
+    # @param max_attempts [Integer] maximum number of retry attempts
+    # @yield the fetch operation to retry
+    # @return [Object] the result of the block
     def fetch_with_auto_retry(max_attempts)
       attempts = 0
 
       begin
         yield
-      rescue => ex
+      rescue => e
         # Tries to auto retry connectivity errors
         raise if attempts >= max_attempts
 
         attempts += 1
 
-        logger.debug { "Retrying fetch attempt #{attempts} for #{ex.message}" }
+        logger.debug { "Retrying fetch attempt #{attempts} for #{e.message}" }
 
         sleep((1..5).to_a.sample)
 
@@ -44,6 +62,11 @@ module Shoryuken
       end
     end
 
+    # Receives messages from an SQS queue
+    #
+    # @param queue [Shoryuken::Polling::QueueConfiguration] the queue configuration
+    # @param limit [Integer] the maximum number of messages to receive
+    # @return [Array<Aws::SQS::Types::Message>, nil] the received messages
     def receive_messages(queue, limit)
       options = receive_options(queue)
 
@@ -58,6 +81,13 @@ module Shoryuken
       shoryuken_queue.receive_messages(options)
     end
 
+    # Determines the maximum number of messages to fetch
+    #
+    # @param shoryuken_queue [Shoryuken::Queue] the queue instance
+    # @param limit [Integer] the requested limit
+    # @param options [Hash] receive options that may contain max_number_of_messages
+    # @option options [Integer] :max_number_of_messages optional override for max messages
+    # @return [Integer] the maximum number of messages to fetch
     def max_number_of_messages(shoryuken_queue, limit, options)
       # For FIFO queues we want to make sure we process one message per group at a time
       # if we set max_number_of_messages greater than 1,
@@ -73,6 +103,10 @@ module Shoryuken
       [limit, FETCH_LIMIT, options[:max_number_of_messages]].compact.min
     end
 
+    # Returns the receive options for a queue
+    #
+    # @param queue [Shoryuken::Polling::QueueConfiguration] the queue configuration
+    # @return [Hash] the receive options hash
     def receive_options(queue)
       options = Shoryuken.sqs_client_receive_message_opts[queue.name]
       options ||= Shoryuken.sqs_client_receive_message_opts[@group]
@@ -80,8 +114,12 @@ module Shoryuken
       options.to_h.dup
     end
 
+    # Checks if a queue uses batch message processing
+    #
+    # @param queue [Shoryuken::Queue] the queue to check
+    # @return [Boolean] true if the queue is configured for batch processing
     def batched_queue?(queue)
       Shoryuken.worker_registry.batch_receive_messages?(queue.name)
     end
   end
-end
+end

data/lib/shoryuken/helpers/atomic_boolean.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  # Helper classes providing thread-safe utilities and data structures
+  module Helpers
+    # A thread-safe boolean implementation using AtomicCounter as base.
+    # Drop-in replacement for Concurrent::AtomicBoolean without external dependencies.
+    # Uses 1 for true and 0 for false internally.
+    class AtomicBoolean < AtomicCounter
+      # Prevent misuse of counter operations on a boolean
+      undef_method :increment, :decrement
+
+      # Initializes a new AtomicBoolean
+      #
+      # @param initial_value [Boolean] the initial value
+      def initialize(initial_value = false)
+        super(initial_value ? 1 : 0)
+      end
+
+      # Gets the current value as a boolean
+      #
+      # @return [Boolean] the current value
+      def value
+        super != 0
+      end
+
+      # Sets the value to true
+      #
+      # @return [Boolean] true
+      def make_true
+        @mutex.synchronize { @value = 1 }
+        true
+      end
+
+      # Sets the value to false
+      #
+      # @return [Boolean] false
+      def make_false
+        @mutex.synchronize { @value = 0 }
+        false
+      end
+
+      # Checks if the value is true
+      #
+      # @return [Boolean] true if the value is true
+      def true?
+        @mutex.synchronize { @value != 0 }
+      end
+
+      # Checks if the value is false
+      #
+      # @return [Boolean] true if the value is false
+      def false?
+        @mutex.synchronize { @value == 0 }
+      end
+    end
+  end
+end

data/lib/shoryuken/helpers/atomic_counter.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # A thread-safe counter implementation using Ruby's Mutex.
+    #
+    # This class provides atomic operations for incrementing, decrementing, and reading
+    # integer values in a thread-safe manner. It serves as a drop-in replacement for
+    # Concurrent::AtomicFixnum without requiring external dependencies.
+    #
+    # The implementation uses a Mutex to ensure thread safety across all Ruby
+    # implementations including JRuby, where true parallelism makes atomic operations
+    # critical for data integrity.
+    #
+    # @note This class is optimized for scenarios with frequent atomic updates
+    #   and occasional reads, such as tracking active worker counts.
+    #
+    # @example Basic usage
+    #   counter = Shoryuken::Helpers::AtomicCounter.new(0)
+    #   counter.increment  # => 1
+    #   counter.increment  # => 2
+    #   counter.value      # => 2
+    #   counter.decrement  # => 1
+    #
+    # @example Tracking busy processors
+    #   @busy_processors = Shoryuken::Helpers::AtomicCounter.new(0)
+    #
+    #   # When starting work
+    #   @busy_processors.increment
+    #
+    #   # When work is done
+    #   @busy_processors.decrement
+    #
+    #   # Check current load
+    #   current_busy = @busy_processors.value
+    class AtomicCounter
+      # Creates a new atomic counter with the specified initial value.
+      #
+      # @param initial_value [Integer] The starting value for the counter
+      # @return [AtomicCounter] A new atomic counter instance
+      #
+      # @example Create counter starting at zero
+      #   counter = Shoryuken::Helpers::AtomicCounter.new
+      #   counter.value  # => 0
+      #
+      # @example Create counter with custom initial value
+      #   counter = Shoryuken::Helpers::AtomicCounter.new(100)
+      #   counter.value  # => 100
+      def initialize(initial_value = 0)
+        @mutex = Mutex.new
+        @value = initial_value
+      end
+
+      # Returns the current value of the counter.
+      #
+      # This operation is thread-safe and will return a consistent value
+      # even when called concurrently with increment/decrement operations.
+      #
+      # @return [Integer] The current counter value
+      #
+      # @example Reading the current value
+      #   counter = Shoryuken::Helpers::AtomicCounter.new(42)
+      #   counter.value  # => 42
+      def value
+        @mutex.synchronize { @value }
+      end
+
+      # Atomically increments the counter by 1 and returns the new value.
+      #
+      # This operation is thread-safe and can be called concurrently from
+      # multiple threads without risk of data corruption or lost updates.
+      #
+      # @return [Integer] The new counter value after incrementing
+      #
+      # @example Incrementing the counter
+      #   counter = Shoryuken::Helpers::AtomicCounter.new(5)
+      #   counter.increment  # => 6
+      #   counter.increment  # => 7
+      def increment
+        @mutex.synchronize { @value += 1 }
+      end
+
+      # Atomically decrements the counter by 1 and returns the new value.
+      #
+      # This operation is thread-safe and can be called concurrently from
+      # multiple threads without risk of data corruption or lost updates.
+      # The counter can go negative if decremented below zero.
+      #
+      # @return [Integer] The new counter value after decrementing
+      #
+      # @example Decrementing the counter
+      #   counter = Shoryuken::Helpers::AtomicCounter.new(5)
+      #   counter.decrement  # => 4
+      #   counter.decrement  # => 3
+      #
+      # @example Counter can go negative
+      #   counter = Shoryuken::Helpers::AtomicCounter.new(0)
+      #   counter.decrement  # => -1
+      def decrement
+        @mutex.synchronize { @value -= 1 }
+      end
+    end
+  end
+end