shoryuken 7.0.0.alpha2 → 7.0.0

data/lib/shoryuken/environment_loader.rb

@@ -1,30 +1,61 @@
 # frozen_string_literal: true
 
 module Shoryuken
+  # Loads and configures the Shoryuken environment from configuration files
+  # and command line options. Handles Rails integration and queue setup.
   class EnvironmentLoader
+    # @return [Hash] the configuration options
     attr_reader :options
 
+    # Sets up a new EnvironmentLoader with the given options
+    #
+    # @param options [Hash] configuration options
+    # @option options [String] :config_file path to the configuration file
+    # @option options [Boolean] :rails whether to initialize Rails
+    # @option options [String] :logfile path to the log file
+    # @option options [Boolean] :verbose whether to enable verbose logging
+    # @option options [String] :require path to require workers from
+    # @option options [Integer] :concurrency number of concurrent workers
+    # @return [Shoryuken::EnvironmentLoader] the configured instance
     def self.setup_options(options)
       instance = new(options)
       instance.setup_options
       instance
     end
 
+    # Loads the environment for Rails console usage
+    #
+    # @return [void]
     def self.load_for_rails_console
       instance = setup_options(config_file: (Rails.root + 'config' + 'shoryuken.yml'))
       instance.load
     end
 
+    # Initializes a new EnvironmentLoader with the given options
+    #
+    # @param options [Hash] configuration options
+    # @option options [String] :config_file path to the configuration file
+    # @option options [Boolean] :rails whether to initialize Rails
+    # @option options [String] :logfile path to the log file
+    # @option options [Boolean] :verbose whether to enable verbose logging
+    # @option options [String] :require path to require workers from
+    # @option options [Integer] :concurrency number of concurrent workers
     def initialize(options)
       @options = options
     end
 
+    # Sets up configuration options from file and initializes components
+    #
+    # @return [void]
     def setup_options
       initialize_rails if load_rails?
       initialize_options
       initialize_logger
     end
 
+    # Loads the environment including queues and workers
+    #
+    # @return [void]
     def load
       prefix_active_job_queue_names
       parse_queues
@@ -35,15 +66,21 @@ module Shoryuken
 
     private
 
+    # Merges configuration file options with runtime options
+    #
+    # @return [void]
     def initialize_options
       Shoryuken.options.merge!(config_file_options)
       Shoryuken.options.merge!(options)
     end
 
+    # Reads and parses the configuration file
+    #
+    # @return [Hash] the parsed configuration options
     def config_file_options
       return {} unless (path = options[:config_file])
 
-      fail ArgumentError, "The supplied config file #{path} does not exist" unless File.exist?(path)
+      raise Errors::InvalidConfigurationError, "The supplied config file #{path} does not exist" unless File.exist?(path)
 
       if (result = YAML.load(ERB.new(IO.read(path)).result))
         Shoryuken::Helpers::HashUtils.deep_symbolize_keys(result)
@@ -52,11 +89,17 @@ module Shoryuken
       end
     end
 
+    # Initializes the logger with file output and verbosity settings
+    #
+    # @return [void]
     def initialize_logger
       Shoryuken::Logging.initialize_logger(Shoryuken.options[:logfile]) if Shoryuken.options[:logfile]
       Shoryuken.logger.level = Logger::DEBUG if Shoryuken.options[:verbose]
     end
 
+    # Initializes the Rails environment
+    #
+    # @return [void]
     def initialize_rails
       # Adapted from: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/cli.rb
 
@@ -80,18 +123,26 @@ module Shoryuken
           end
         end
         if Shoryuken.active_job?
-          require 'shoryuken/extensions/active_job_extensions'
-          require 'shoryuken/extensions/active_job_adapter'
-          require 'shoryuken/extensions/active_job_concurrent_send_adapter'
+          require 'active_job/extensions'
+          require 'active_job/queue_adapters/shoryuken_adapter'
+          require 'active_job/queue_adapters/shoryuken_concurrent_send_adapter'
         end
         require File.expand_path('config/environment.rb')
       end
     end
 
+    # Checks if Rails should be loaded
+    #
+    # @return [Boolean] true if Rails should be initialized
     def load_rails?
       options[:rails]
     end
 
+    # Prefixes a queue name with ActiveJob queue name prefix
+    #
+    # @param queue_name [String] the queue name to prefix
+    # @param weight [Integer] the queue weight
+    # @return [Array<String, Integer>] the prefixed queue name and weight
     def prefix_active_job_queue_name(queue_name, weight)
       return [queue_name, weight] if queue_name.start_with?('https://', 'arn:')
 
@@ -104,6 +155,9 @@ module Shoryuken
       [prefixed_queue_name, weight]
     end
 
+    # Prefixes all queue names with ActiveJob prefix if enabled
+    #
+    # @return [void]
     def prefix_active_job_queue_names
       return unless Shoryuken.active_job?
       return unless Shoryuken.active_job_queue_name_prefixing?
@@ -123,10 +177,19 @@ module Shoryuken
       end
     end
 
+    # Parses a single queue and adds it to a group
+    #
+    # @param queue [String] the queue name
+    # @param weight [Integer] the queue weight
+    # @param group [String] the group name
+    # @return [void]
     def parse_queue(queue, weight, group)
       Shoryuken.add_queue(queue, [weight.to_i, 1].max, group)
     end
 
+    # Parses all queues from configuration and adds them to groups
+    #
+    # @return [void]
     def parse_queues
       if Shoryuken.options[:queues].to_a.any?
         Shoryuken.add_group('default', Shoryuken.options[:concurrency])
@@ -145,6 +208,9 @@ module Shoryuken
       end
     end
 
+    # Requires worker files from the configured path
+    #
+    # @return [void]
     def require_workers
       required = Shoryuken.options[:require]
 
@@ -157,6 +223,10 @@ module Shoryuken
       end
     end
 
+    # Validates that all configured queues exist in SQS
+    #
+    # @return [void]
+    # @raise [ArgumentError] if any queues do not exist
     def validate_queues
       return Shoryuken.logger.warn { 'No queues supplied' } if Shoryuken.ungrouped_queues.empty?
 
@@ -178,12 +248,12 @@ module Shoryuken
         It's also possible that you don't have permission to access the specified queues.
       MSG
 
-      fail(
-        ArgumentError,
-        error_msg
-      )
+      raise Errors::QueueNotFoundError, error_msg
     end
 
+    # Validates that all queues have registered workers
+    #
+    # @return [void]
     def validate_workers
       return if Shoryuken.active_job?
 

data/lib/shoryuken/errors.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  # Namespace for all Shoryuken-specific errors.
+  # These provide more meaningful error types than generic Ruby exceptions,
+  # making it easier to rescue and handle specific failure cases.
+  module Errors
+    # Base class for all Shoryuken errors
+    BaseError = Class.new(StandardError)
+
+    # Raised when there is a configuration validation failure
+    InvalidConfigurationError = Class.new(BaseError)
+
+    # Raised when a specified SQS queue does not exist or cannot be accessed
+    QueueNotFoundError = Class.new(BaseError)
+
+    # Raised when worker registration fails due to conflicts
+    # (e.g., registering multiple workers for a batch queue)
+    InvalidWorkerRegistrationError = Class.new(BaseError)
+
+    # Raised when an invalid polling strategy is specified
+    InvalidPollingStrategyError = Class.new(BaseError)
+
+    # Raised when an invalid lifecycle event name is used
+    InvalidEventError = Class.new(BaseError)
+
+    # Raised when a delay exceeds the maximum allowed by SQS (15 minutes)
+    InvalidDelayError = Class.new(BaseError)
+
+    # Raised when an ARN format is invalid
+    InvalidArnError = Class.new(BaseError)
+  end
+end

data/lib/shoryuken/fetcher.rb

@@ -1,15 +1,26 @@
 # frozen_string_literal: true
 
 module Shoryuken
+  # Fetches messages from SQS queues.
+  # Handles message retrieval with automatic retry on connectivity errors.
   class Fetcher
     include Util
 
+    # Maximum number of messages that can be fetched in a single SQS request
     FETCH_LIMIT = 10
 
+    # Initializes a new Fetcher for a processing group
+    #
+    # @param group [String] the processing group name
     def initialize(group)
       @group = group
     end
 
+    # Fetches messages from a queue with automatic retry
+    #
+    # @param queue [Shoryuken::Polling::QueueConfiguration] the queue configuration
+    # @param limit [Integer] the maximum number of messages to fetch
+    # @return [Array<Aws::SQS::Types::Message>] the fetched messages
     def fetch(queue, limit)
       fetch_with_auto_retry(3) do
         started_at = Time.now
@@ -27,6 +38,11 @@ module Shoryuken
 
     private
 
+    # Fetches with automatic retry on errors
+    #
+    # @param max_attempts [Integer] maximum number of retry attempts
+    # @yield the fetch operation to retry
+    # @return [Object] the result of the block
     def fetch_with_auto_retry(max_attempts)
       attempts = 0
 
@@ -46,6 +62,11 @@ module Shoryuken
       end
     end
 
+    # Receives messages from an SQS queue
+    #
+    # @param queue [Shoryuken::Polling::QueueConfiguration] the queue configuration
+    # @param limit [Integer] the maximum number of messages to receive
+    # @return [Array<Aws::SQS::Types::Message>, nil] the received messages
     def receive_messages(queue, limit)
       options = receive_options(queue)
 
@@ -60,6 +81,13 @@ module Shoryuken
       shoryuken_queue.receive_messages(options)
     end
 
+    # Determines the maximum number of messages to fetch
+    #
+    # @param shoryuken_queue [Shoryuken::Queue] the queue instance
+    # @param limit [Integer] the requested limit
+    # @param options [Hash] receive options that may contain max_number_of_messages
+    # @option options [Integer] :max_number_of_messages optional override for max messages
+    # @return [Integer] the maximum number of messages to fetch
     def max_number_of_messages(shoryuken_queue, limit, options)
       # For FIFO queues we want to make sure we process one message per group at a time
       # if we set max_number_of_messages greater than 1,
@@ -75,6 +103,10 @@ module Shoryuken
       [limit, FETCH_LIMIT, options[:max_number_of_messages]].compact.min
     end
 
+    # Returns the receive options for a queue
+    #
+    # @param queue [Shoryuken::Polling::QueueConfiguration] the queue configuration
+    # @return [Hash] the receive options hash
     def receive_options(queue)
       options = Shoryuken.sqs_client_receive_message_opts[queue.name]
       options ||= Shoryuken.sqs_client_receive_message_opts[@group]
@@ -82,8 +114,12 @@ module Shoryuken
       options.to_h.dup
     end
 
+    # Checks if a queue uses batch message processing
+    #
+    # @param queue [Shoryuken::Queue] the queue to check
+    # @return [Boolean] true if the queue is configured for batch processing
     def batched_queue?(queue)
       Shoryuken.worker_registry.batch_receive_messages?(queue.name)
     end
   end
-end
+end

data/lib/shoryuken/helpers/atomic_boolean.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Shoryuken
+  # Helper classes providing thread-safe utilities and data structures
   module Helpers
     # A thread-safe boolean implementation using AtomicCounter as base.
     # Drop-in replacement for Concurrent::AtomicBoolean without external dependencies.
@@ -9,33 +10,46 @@ module Shoryuken
       # Prevent misuse of counter operations on a boolean
       undef_method :increment, :decrement
 
+      # Initializes a new AtomicBoolean
+      #
+      # @param initial_value [Boolean] the initial value
       def initialize(initial_value = false)
         super(initial_value ? 1 : 0)
       end
 
-      # Get the current value as boolean
+      # Gets the current value as a boolean
+      #
+      # @return [Boolean] the current value
       def value
         super != 0
       end
 
-      # Set the value to true
+      # Sets the value to true
+      #
+      # @return [Boolean] true
       def make_true
         @mutex.synchronize { @value = 1 }
         true
       end
 
-      # Set the value to false
+      # Sets the value to false
+      #
+      # @return [Boolean] false
       def make_false
         @mutex.synchronize { @value = 0 }
         false
       end
 
-      # Check if the value is true
+      # Checks if the value is true
+      #
+      # @return [Boolean] true if the value is true
       def true?
         @mutex.synchronize { @value != 0 }
       end
 
-      # Check if the value is false
+      # Checks if the value is false
+      #
+      # @return [Boolean] true if the value is false
       def false?
         @mutex.synchronize { @value == 0 }
       end

data/lib/shoryuken/helpers/timer_task.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # A thread-safe timer task implementation.
+    # Drop-in replacement for Concurrent::TimerTask without external dependencies.
+    class TimerTask
+      # Initializes a new TimerTask
+      #
+      # @param execution_interval [Float] interval in seconds between task executions
+      # @param task [Proc] the task to execute on each interval (provided as a block)
+      # @return [TimerTask] a new TimerTask instance
+      # @raise [ArgumentError] if no block is provided or interval is not positive
+      # @yield the task to execute on each interval
+      def initialize(execution_interval:, &task)
+        raise ArgumentError, 'A block must be provided' unless block_given?
+
+        @execution_interval = Float(execution_interval)
+        raise ArgumentError, 'execution_interval must be positive' if @execution_interval <= 0
+
+        @task = task
+        @mutex = Mutex.new
+        @thread = nil
+        @running = false
+        @killed = false
+      end
+
+      # Starts the timer task execution
+      #
+      # @return [TimerTask] self for method chaining
+      def execute
+        @mutex.synchronize do
+          return self if @running || @killed
+
+          @running = true
+          @thread = Thread.new { run_timer_loop }
+        end
+        self
+      end
+
+      # Stops and kills the timer task
+      #
+      # @return [Boolean] true if killed, false if already killed
+      def kill
+        @mutex.synchronize do
+          return false if @killed
+
+          @killed = true
+          @running = false
+
+          @thread.kill if @thread&.alive?
+        end
+        true
+      end
+
+      private
+
+      # Runs the timer loop in a separate thread
+      #
+      # @return [void]
+      def run_timer_loop
+        until @killed
+          sleep(@execution_interval)
+          break if @killed
+
+          begin
+            @task.call
+          rescue => e
+            # Log the error but continue running
+            # This matches the behavior of Concurrent::TimerTask
+            warn "TimerTask execution error: #{e.message}"
+            warn e.backtrace.join("\n") if e.backtrace
+          end
+        end
+      ensure
+        @mutex.synchronize { @running = false }
+      end
+    end
+  end
+end

data/lib/shoryuken/launcher.rb

@@ -1,13 +1,31 @@
 # frozen_string_literal: true
 
 module Shoryuken
+  # Launches and coordinates Shoryuken's message processing managers.
+  # Handles the lifecycle of the processing system including startup, shutdown, and health checks.
   class Launcher
     include Util
 
+    # Initializes a new Launcher with managers for each processing group
     def initialize
       @managers = create_managers
+      @stopping = false
     end
 
+    # Indicates whether the launcher is in the process of stopping.
+    #
+    # This flag is set to true when either {#stop} or {#stop!} is called,
+    # and is used by ActiveJob adapters to signal jobs that they should
+    # checkpoint and prepare for graceful shutdown.
+    #
+    # @return [Boolean] true if stopping, false otherwise
+    def stopping?
+      @stopping
+    end
+
+    # Starts the message processing system
+    #
+    # @return [void]
     def start
       logger.info { 'Starting' }
 
@@ -15,7 +33,11 @@ module Shoryuken
       start_managers
     end
 
+    # Forces an immediate stop of all processing
+    #
+    # @return [void]
     def stop!
+      @stopping = true
       initiate_stop
 
       # Don't await here so the timeout below is not delayed
@@ -27,7 +49,11 @@ module Shoryuken
       fire_event(:stopped)
     end
 
+    # Gracefully stops all processing, waiting for in-flight messages
+    #
+    # @return [void]
     def stop
+      @stopping = true
       fire_event(:quiet, true)
 
       initiate_stop
@@ -41,6 +67,9 @@ module Shoryuken
       fire_event(:stopped)
     end
 
+    # Checks if all processing groups are healthy
+    #
+    # @return [Boolean] true if all groups are running normally
     def healthy?
       Shoryuken.groups.keys.all? do |group|
         manager = @managers.find { |m| m.group == group }
@@ -50,30 +79,48 @@ module Shoryuken
 
     private
 
+    # Stops all managers from dispatching new messages
+    #
+    # @return [void]
     def stop_new_dispatching
       @managers.each(&:stop_new_dispatching)
     end
 
+    # Waits for any in-progress dispatching to complete
+    #
+    # @return [void]
     def await_dispatching_in_progress
       @managers.each(&:await_dispatching_in_progress)
     end
 
+    # Returns the executor for running async operations
+    #
+    # @return [Concurrent::ExecutorService] the executor service
     def executor
       @_executor ||= Shoryuken.launcher_executor || Concurrent.global_io_executor
     end
 
+    # Starts all managers in parallel futures
+    #
+    # @return [void]
     def start_managers
       @managers.each do |manager|
         Concurrent::Future.execute { manager.start }
       end
     end
 
+    # Initiates the stop sequence
+    #
+    # @return [void]
     def initiate_stop
       logger.info { 'Shutting down' }
 
       stop_callback
     end
 
+    # Executes the start callback and fires startup event
+    #
+    # @return [void]
     def start_callback
       if (callback = Shoryuken.start_callback)
         logger.debug { 'Calling start_callback' }
@@ -83,6 +130,9 @@ module Shoryuken
       fire_event(:startup)
     end
 
+    # Executes the stop callback and fires shutdown event
+    #
+    # @return [void]
     def stop_callback
       if (callback = Shoryuken.stop_callback)
         logger.debug { 'Calling stop_callback' }
@@ -92,6 +142,9 @@ module Shoryuken
       fire_event(:shutdown, true)
     end
 
+    # Creates managers for each configured processing group
+    #
+    # @return [Array<Shoryuken::Manager>] the created managers
     def create_managers
       Shoryuken.groups.map do |group, options|
         Shoryuken::Manager.new(

data/lib/shoryuken/logging/base.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Logging
+    # Base formatter class that provides common functionality for Shoryuken log formatters.
+    # Provides thread ID generation and context management.
+    class Base < ::Logger::Formatter
+      # Generates a thread ID for the current thread.
+      # Uses a combination of thread object_id and process ID to create a unique identifier.
+      #
+      # @return [String] A base36-encoded thread identifier
+      def tid
+        Thread.current['shoryuken_tid'] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
+      end
+
+      # Returns the current logging context as a formatted string.
+      # Context is set using {Shoryuken::Logging.with_context}.
+      #
+      # @return [String] Formatted context string or empty string if no context
+      def context
+        c = Shoryuken::Logging.current_context
+        c ? " #{c}" : ''
+      end
+    end
+  end
+end

data/lib/shoryuken/logging/pretty.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Logging
+    # A pretty log formatter that includes timestamps, process ID, thread ID,
+    # context information, and severity in a human-readable format.
+    #
+    # Output format: "TIMESTAMP PID TID-THREAD_ID CONTEXT SEVERITY: MESSAGE"
+    #
+    # @example Sample output
+    #   # 2023-08-15T10:30:45Z 12345 TID-abc123 MyWorker/queue1/msg-456 INFO: Processing message
+    class Pretty < Base
+      # Formats a log message with timestamp and full context information.
+      #
+      # @param severity [String] Log severity level (DEBUG, INFO, WARN, ERROR, FATAL)
+      # @param time [Time] Timestamp when the log entry was created
+      # @param _program_name [String] Program name (unused)
+      # @param message [String] The log message
+      # @return [String] Formatted log entry with newline
+      def call(severity, time, _program_name, message)
+        "#{time.utc.iso8601} #{Process.pid} TID-#{tid}#{context} #{severity}: #{message}\n"
+      end
+    end
+  end
+end

data/lib/shoryuken/logging/without_timestamp.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Logging
+    # A log formatter that excludes timestamps from output.
+    # Useful for environments where timestamps are added by external logging systems.
+    #
+    # Output format: "pid=PID tid=THREAD_ID CONTEXT SEVERITY: MESSAGE"
+    #
+    # @example Sample output
+    #   # pid=12345 tid=abc123 MyWorker/queue1/msg-456 INFO: Processing message
+    class WithoutTimestamp < Base
+      # Formats a log message without timestamp information.
+      #
+      # @param severity [String] Log severity level (DEBUG, INFO, WARN, ERROR, FATAL)
+      # @param _time [Time] Timestamp (unused)
+      # @param _program_name [String] Program name (unused)
+      # @param message [String] The log message
+      # @return [String] Formatted log entry with newline
+      def call(severity, _time, _program_name, message)
+        "pid=#{Process.pid} tid=#{tid}#{context} #{severity}: #{message}\n"
+      end
+    end
+  end
+end