shoryuken 6.2.1 → 7.0.2

data/lib/shoryuken/middleware/entry.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Middleware
+    # Represents an entry in a middleware chain, storing the middleware class
+    # and any arguments needed for its instantiation.
+    #
+    # @api private
+    class Entry
+      # @return [Class] The middleware class this entry represents
+      attr_reader :klass
+
+      # Creates a new middleware entry.
+      #
+      # @param klass [Class] The middleware class
+      # @param args [Array] Arguments to pass to the middleware constructor
+      def initialize(klass, *args)
+        @klass = klass
+        @args  = args
+      end
+
+      # Creates a new instance of the middleware class with the stored arguments.
+      #
+      # @return [Object] A new instance of the middleware class
+      def make_new
+        @klass.new(*@args)
+      end
+    end
+  end
+end

data/lib/shoryuken/middleware/server/active_record.rb

@@ -1,7 +1,17 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
+    # Server-side middleware that runs around message processing
     module Server
+      # Middleware that clears ActiveRecord connections after message processing.
+      # Ensures database connections are returned to the pool after each job.
       class ActiveRecord
+        # Processes a message and clears database connections afterwards
+        #
+        # @param _args [Array<Object>] middleware call arguments (unused)
+        # @yield continues to the next middleware in the chain
+        # @return [Object] return value from the next middleware or worker in the chain
         def call(*_args)
           yield
         ensure

data/lib/shoryuken/middleware/server/auto_delete.rb

@@ -1,7 +1,19 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server
+      # Middleware that automatically deletes messages after successful processing.
+      # Only deletes messages when the worker has auto_delete enabled.
       class AutoDelete
+        # Processes a message and deletes it if auto_delete is enabled
+        #
+        # @param worker [Object] the worker instance
+        # @param queue [String] the queue name
+        # @param sqs_msg [Shoryuken::Message, Array<Shoryuken::Message>] the message or batch
+        # @param _body [Object] the parsed message body (unused)
+        # @yield continues to the next middleware in the chain
+        # @return [void]
         def call(worker, queue, sqs_msg, _body)
           yield
 

data/lib/shoryuken/middleware/server/auto_extend_visibility.rb

@@ -1,11 +1,24 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server
+      # Middleware that automatically extends message visibility timeout during processing.
+      # Prevents messages from becoming visible to other consumers while still being processed.
       class AutoExtendVisibility
         include Util
 
+        # Number of seconds before timeout to extend visibility
         EXTEND_UPFRONT_SECONDS = 5
 
+        # Processes a message with automatic visibility timeout extension
+        #
+        # @param worker [Object] the worker instance
+        # @param queue [String] the queue name
+        # @param sqs_msg [Shoryuken::Message, Array<Shoryuken::Message>] the message or batch
+        # @param body [Object] the parsed message body
+        # @yield continues to the next middleware in the chain
+        # @return [void]
         def call(worker, queue, sqs_msg, body)
           return yield unless worker.class.auto_visibility_timeout?
 
@@ -22,28 +35,41 @@ module Shoryuken
 
         private
 
+        # Helper class for extending message visibility
         class MessageVisibilityExtender
           include Util
 
+          # Creates a timer task that extends message visibility
+          #
+          # @param _worker [Object] the worker instance (unused)
+          # @param queue [String] the queue name
+          # @param sqs_msg [Shoryuken::Message] the message
+          # @param _body [Object] the parsed message body (unused)
+          # @return [Shoryuken::Helpers::TimerTask] the timer task
           def auto_extend(_worker, queue, sqs_msg, _body)
             queue_visibility_timeout = Shoryuken::Client.queues(queue).visibility_timeout
 
-            Concurrent::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
-              begin
-                logger.debug do
-                  "Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
-                end
-
-                sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
-              rescue => ex
-                logger.error do
-                  "Could not auto extend the message #{queue}/#{sqs_msg.message_id} visibility timeout. Error: #{ex.message}"
-                end
+            Shoryuken::Helpers::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
+              logger.debug do
+                "Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
+              end
+
+              sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
+            rescue => e
+              logger.error do
+                "Could not auto extend the message #{queue}/#{sqs_msg.message_id} visibility timeout. Error: #{e.message}"
               end
             end
           end
         end
 
+        # Creates and starts a visibility extension timer
+        #
+        # @param worker [Object] the worker instance
+        # @param queue [String] the queue name
+        # @param sqs_msg [Shoryuken::Message] the message
+        # @param body [Object] the parsed message body
+        # @return [Shoryuken::Helpers::TimerTask] the started timer
         def auto_visibility_timer(worker, queue, sqs_msg, body)
           MessageVisibilityExtender.new.auto_extend(worker, queue, sqs_msg, body).tap(&:execute)
         end

data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb

@@ -1,9 +1,24 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server
+      # Middleware that implements exponential backoff retry for failed messages.
+      # When a job fails, the message visibility timeout is adjusted based on
+      # configured retry intervals.
       class ExponentialBackoffRetry
         include Util
 
+        # Processes a message with exponential backoff retry on failure
+        #
+        # @param worker [Object] the worker instance
+        # @param _queue [String] the queue name (unused)
+        # @param sqs_msg [Shoryuken::Message, Array<Shoryuken::Message>] the message or batch
+        # @param _body [Object] the parsed message body (unused)
+        # @yield continues to the next middleware in the chain
+        # @return [void]
+        # @raise [StandardError] re-raises the original exception if retry intervals are not configured
+        #   or if retry limit is exceeded
         def call(worker, _queue, sqs_msg, _body)
           return yield unless worker.class.exponential_backoff?
 
@@ -14,7 +29,7 @@ module Shoryuken
 
           started_at = Time.now
           yield
-        rescue => ex
+        rescue => e
           retry_intervals = worker.class.get_shoryuken_options['retry_intervals']
 
           if retry_intervals.nil? || !handle_failure(sqs_msg, started_at, retry_intervals)
@@ -23,13 +38,18 @@ module Shoryuken
             raise
           end
 
-          logger.warn { "Message #{sqs_msg.message_id} will attempt retry due to error: #{ex.message}" }
+          logger.warn { "Message #{sqs_msg.message_id} will attempt retry due to error: #{e.message}" }
           # since we didn't raise, lets log the backtrace for debugging purposes.
-          logger.debug { ex.backtrace.join("\n") } unless ex.backtrace.nil?
+          logger.debug { e.backtrace.join("\n") } unless e.backtrace.nil?
         end
 
         private
 
+        # Gets the retry interval for a given attempt number
+        #
+        # @param retry_intervals [Array<Integer>, #call] the configured intervals or callable
+        # @param attempts [Integer] the current attempt number
+        # @return [Integer, nil] the interval in seconds or nil
         def get_interval(retry_intervals, attempts)
           return retry_intervals.call(attempts) if retry_intervals.respond_to?(:call)
 
@@ -40,12 +60,23 @@ module Shoryuken
           end
         end
 
+        # Calculates the next visibility timeout capped at SQS maximum
+        #
+        # @param interval [Integer] the desired interval
+        # @param started_at [Time] when processing started
+        # @return [Integer] the capped visibility timeout
         def next_visibility_timeout(interval, started_at)
           max_timeout = 43_200 - (Time.now - started_at).ceil - 1
           interval = max_timeout if interval > max_timeout
           interval.to_i
         end
 
+        # Handles a message failure by adjusting visibility timeout
+        #
+        # @param sqs_msg [Shoryuken::Message] the failed message
+        # @param started_at [Time] when processing started
+        # @param retry_intervals [Array<Integer>, #call] the configured intervals
+        # @return [Boolean] true if retry was scheduled, false otherwise
         def handle_failure(sqs_msg, started_at, retry_intervals)
           receive_count = sqs_msg.attributes['ApproximateReceiveCount'].to_i
 

data/lib/shoryuken/middleware/server/non_retryable_exception.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Middleware
+    module Server
+      # Middleware that handles non-retryable exceptions by deleting messages immediately.
+      # When a configured exception occurs, the message is deleted instead of being retried.
+      #
+      # Configure non-retryable exceptions per worker:
+      #
+      #   class MyWorker
+      #     include Shoryuken::Worker
+      #
+      #     # Using exception classes
+      #     shoryuken_options queue: 'my_queue',
+      #                       non_retryable_exceptions: [InvalidInputError, RecordNotFoundError]
+      #
+      #     # Or using a lambda for dynamic classification
+      #     shoryuken_options queue: 'my_queue',
+      #                       non_retryable_exceptions: ->(error) {
+      #                         error.is_a?(StandardError) && error.message.include?('permanent')
+      #                       }
+      #
+      #     def perform(sqs_msg, body)
+      #       # ...
+      #     end
+      #   end
+      class NonRetryableException
+        include Util
+
+        # Processes a message and handles non-retryable exceptions
+        #
+        # @param worker [Object] the worker instance
+        # @param queue [String] the queue name
+        # @param sqs_msg [Shoryuken::Message, Array<Shoryuken::Message>] the message or batch
+        # @param _body [Object] the parsed message body (unused)
+        # @yield continues to the next middleware in the chain
+        # @return [void]
+        def call(worker, queue, sqs_msg, _body)
+          yield
+        rescue => e
+          non_retryable_exceptions = worker.class.get_shoryuken_options['non_retryable_exceptions']
+
+          return raise unless non_retryable_exceptions
+
+          if non_retryable_exceptions.respond_to?(:call)
+            return raise unless non_retryable_exceptions.call(e)
+          else
+            exception_classes = Array(non_retryable_exceptions)
+            return raise unless exception_classes.any? { |klass| e.is_a?(klass) }
+          end
+
+          # Handle batch messages
+          messages = sqs_msg.is_a?(Array) ? sqs_msg : [sqs_msg]
+
+          logger.warn do
+            "Non-retryable exception #{e.class} occurred for message(s) #{messages.map(&:message_id).join(', ')}. " \
+            "Deleting message(s) immediately. Error: #{e.message}"
+          end
+
+          logger.debug { e.backtrace.join("\n") } if e.backtrace
+
+          # Delete the message(s) immediately
+          entries = messages.map.with_index { |message, i| { id: i.to_s, receipt_handle: message.receipt_handle } }
+
+          begin
+            queue_client = Shoryuken::Client.queues(queue)
+            delete_failed = queue_client.delete_messages(entries: entries)
+
+            # Check if deletion reported failures (returns true if any failed)
+            if delete_failed
+              logger.warn do
+                'Failed to delete some messages for non-retryable exception on queue ' \
+                  "'#{queue}'. " \
+                  "Entries: #{entries.map { |e| { id: e[:id] } }.inspect}. " \
+                  'Some messages may remain in the queue and could be reprocessed.'
+              end
+            end
+          rescue => delete_error
+            logger.error do
+              'Error deleting messages for non-retryable exception on queue ' \
+                "'#{queue}': #{delete_error.class} - #{delete_error.message}. " \
+                "Entries: #{entries.map { |e| { id: e[:id] } }.inspect}. " \
+                'Messages may remain in the queue and could be reprocessed.'
+            end
+            logger.debug { delete_error.backtrace.join("\n") } if delete_error.backtrace
+          end
+
+          # Don't re-raise - the exception has been handled by deleting the message
+        end
+      end
+    end
+  end
+end
+

data/lib/shoryuken/middleware/server/timing.rb

@@ -1,9 +1,22 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server
+      # Middleware that logs timing information for message processing.
+      # Records start time, completion time, and warns if processing
+      # exceeds the queue's visibility timeout.
       class Timing
         include Util
 
+        # Processes a message while logging timing information
+        #
+        # @param _worker [Object] the worker instance (unused)
+        # @param queue [String] the queue name
+        # @param _sqs_msg [Shoryuken::Message] the message being processed (unused)
+        # @param _body [Object] the parsed message body (unused)
+        # @yield continues to the next middleware in the chain
+        # @return [void]
         def call(_worker, queue, _sqs_msg, _body)
           started_at = Time.now
 

data/lib/shoryuken/options.rb

@@ -1,6 +1,13 @@
+# frozen_string_literal: true
+
 module Shoryuken
+  # Stores and manages all Shoryuken configuration options.
+  # This class is used internally to hold settings for workers, queues,
+  # middleware, and other runtime configurations.
   class Options
+    # Default configuration values for Shoryuken
     DEFAULTS = {
+      thread_priority: -1,
       concurrency: 25,
       queues: [],
       aws: {},
@@ -16,12 +23,31 @@ module Shoryuken
       }
     }.freeze
 
-    attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout, :groups,
-                  :launcher_executor, :reloader, :enable_reloading,
-                  :start_callback, :stop_callback, :worker_executor, :worker_registry, :exception_handlers
-    attr_writer :default_worker_options, :sqs_client
+    # @return [Boolean] whether to enable ActiveJob queue name prefixing
+    # @return [Boolean] whether to cache SQS visibility timeout
+    # @return [Hash{String => Hash}] the configured processing groups
+    # @return [Object] the executor used to launch workers
+    # @return [Proc] the code reloader proc for development environments
+    # @return [Boolean] whether code reloading is enabled
+    # @return [Proc, nil] callback to execute when server starts
+    # @return [Proc, nil] callback to execute when server stops
+    # @return [Class] the executor class for running workers
+    # @return [Shoryuken::WorkerRegistry] the registry for worker classes
+    # @return [Array<#call>] handlers for processing exceptions
+    attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout,
+                  :groups, :launcher_executor, :reloader, :enable_reloading,
+                  :start_callback, :stop_callback, :worker_executor, :worker_registry,
+                  :exception_handlers
+
+    # @return [Hash] the default options for workers
+    # @return [Aws::SQS::Client] the SQS client instance
+    # @return [Logger] the logger instance
+    attr_writer :default_worker_options, :sqs_client, :logger
+
+    # @return [Hash] options passed to SQS receive_message calls
     attr_reader :sqs_client_receive_message_opts
 
+    # Initializes a new Options instance with default values
     def initialize
       self.groups = {}
       self.worker_registry = DefaultWorkerRegistry.new
@@ -35,33 +61,63 @@ module Shoryuken
       @sqs_client_receive_message_opts ||= {}
     end
 
+    # Checks if ActiveJob is available
+    #
+    # @return [Boolean] true if ActiveJob is defined
     def active_job?
       defined?(::ActiveJob)
     end
 
-    def add_group(group, concurrency = nil, delay: nil)
+    # Adds a processing group with the specified concurrency and delay
+    #
+    # @param group [String] the name of the group
+    # @param concurrency [Integer, nil] the number of concurrent workers for the group
+    # @param delay [Float, nil] the delay between polling cycles
+    # @param polling_strategy [Class, String, nil] the polling strategy class for the group
+    # @return [Hash] the group configuration
+    # @raise [Errors::InvalidPollingStrategyError] if polling_strategy is not a Class, String, or nil
+    def add_group(group, concurrency = nil, delay: nil, polling_strategy: nil)
+      unless polling_strategy.nil? || polling_strategy.is_a?(Class) || polling_strategy.is_a?(String)
+        raise Errors::InvalidPollingStrategyError, "#{polling_strategy} is not a valid polling_strategy"
+      end
+
       concurrency ||= options[:concurrency]
       delay ||= options[:delay]
 
       groups[group] ||= {
         concurrency: concurrency,
         delay: delay,
+        polling_strategy: polling_strategy,
         queues: []
       }
     end
 
+    # Adds a queue to a processing group with the specified weight
+    #
+    # @param queue [String] the name of the queue
+    # @param weight [Integer] the weight (priority) of the queue
+    # @param group [String] the name of the group to add the queue to
+    # @return [void]
     def add_queue(queue, weight, group)
       weight.times do
         groups[group][:queues] << queue
       end
     end
 
+    # Returns all queues from all groups
+    #
+    # @return [Array<String>] flat array of all queue names
     def ungrouped_queues
       groups.values.flat_map { |options| options[:queues] }
     end
 
+    # Returns the polling strategy class for a group
+    #
+    # @param group [String] the name of the group
+    # @return [Class] the polling strategy class to use
     def polling_strategy(group)
-      strategy = (group == 'default' ? options : options[:groups].to_h[group]).to_h[:polling_strategy]
+      strategy = groups[group].to_h[:polling_strategy] ||
+                 (group == 'default' ? options : options[:groups].to_h[group]).to_h[:polling_strategy]
       case strategy
       when 'WeightedRoundRobin', nil # Default case
         Polling::WeightedRoundRobin
@@ -71,57 +127,111 @@ module Shoryuken
         begin
           Object.const_get(strategy)
         rescue NameError
-          raise ArgumentError, "#{strategy} is not a valid polling_strategy"
+          raise Errors::InvalidPollingStrategyError, "#{strategy} is not a valid polling_strategy"
         end
       when Class
         strategy
       end
     end
 
+    # Returns the polling delay for a group
+    #
+    # @param group [String] the name of the group
+    # @return [Float] the delay in seconds
     def delay(group)
       groups[group].to_h.fetch(:delay, options[:delay]).to_f
     end
 
+    # Returns the SQS client, initializing a default one if needed.
+    # Uses AWS configuration from options[:aws] if available.
+    #
+    # @return [Aws::SQS::Client] the SQS client
     def sqs_client
-      @sqs_client ||= Aws::SQS::Client.new
+      @sqs_client ||= Aws::SQS::Client.new(options[:aws])
     end
 
+    # Sets the SQS client receive message options for the default group
+    #
+    # @param sqs_client_receive_message_opts [Hash] the options hash
+    # @return [Hash] the options hash
     def sqs_client_receive_message_opts=(sqs_client_receive_message_opts)
       @sqs_client_receive_message_opts['default'] = sqs_client_receive_message_opts
     end
 
+    # Returns the global options hash
+    #
+    # @return [Hash] the options hash
     def options
       @options ||= DEFAULTS.dup
     end
 
+    # Returns the logger instance
+    #
+    # @return [Logger] the logger
     def logger
-      Shoryuken::Logging.logger
+      @logger ||= Shoryuken::Logging.logger
     end
 
+    # Returns the thread priority setting
+    #
+    # @return [Integer] the thread priority
+    def thread_priority
+      @thread_priority ||= options[:thread_priority]
+    end
+
+    # Sets the thread priority
+    #
+    # @param value [Integer] the thread priority value
+    # @return [Integer] the thread priority
+    attr_writer :thread_priority
+
+    # Registers a worker class with the worker registry
+    #
+    # @param args [Array] arguments to pass to the registry
+    # @return [void]
     def register_worker(*args)
       worker_registry.register_worker(*args)
     end
 
+    # Yields self if running as a server for server-specific configuration
+    #
+    # @yield [Shoryuken::Options] the options instance
+    # @return [void]
     def configure_server
       yield self if server?
     end
 
+    # Returns the server middleware chain
+    #
+    # @yield [Shoryuken::Middleware::Chain] the middleware chain for configuration
+    # @return [Shoryuken::Middleware::Chain] the server middleware chain
     def server_middleware
       @_server_chain ||= default_server_middleware
       yield @_server_chain if block_given?
       @_server_chain
     end
 
+    # Yields self unless running as a server for client-specific configuration
+    #
+    # @yield [Shoryuken::Options] the options instance
+    # @return [void]
     def configure_client
       yield self unless server?
     end
 
+    # Returns the client middleware chain
+    #
+    # @yield [Shoryuken::Middleware::Chain] the middleware chain for configuration
+    # @return [Shoryuken::Middleware::Chain] the client middleware chain
     def client_middleware
       @_client_chain ||= default_client_middleware
       yield @_client_chain if block_given?
       @_client_chain
     end
 
+    # Returns the default worker options hash
+    #
+    # @return [Hash{String => Object}] the default worker options
     def default_worker_options
       @default_worker_options ||= {
         'queue' => 'default',
@@ -133,46 +243,74 @@ module Shoryuken
       }
     end
 
+    # Registers a callback to run when the server starts
+    #
+    # @param block [Proc] the block to execute on start
+    # @return [void]
+    # @yield the block to execute on start
     def on_start(&block)
       self.start_callback = block
     end
 
+    # Registers a callback to run when the server stops
+    #
+    # @param block [Proc] the block to execute on stop
+    # @return [void]
+    # @yield the block to execute on stop
     def on_stop(&block)
       self.stop_callback = block
     end
 
-    # Register a block to run at a point in the Shoryuken lifecycle.
-    # :startup, :quiet, :shutdown or :stopped are valid events.
+    # Registers a block to run at a point in the Shoryuken lifecycle.
     #
+    # @param event [Symbol] the lifecycle event (:startup, :quiet, :shutdown, or :stopped)
+    # @param block [Proc] the block to execute for the event
+    # @return [void]
+    # @raise [ArgumentError] if event is not a Symbol or not a valid event name
+    # @yield the block to execute for the event
+    # @example
     #   Shoryuken.configure_server do |config|
     #     config.on(:shutdown) do
     #       puts "Goodbye cruel world!"
     #     end
     #   end
     def on(event, &block)
-      fail ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
-      fail ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
+      raise Errors::InvalidEventError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
+      raise Errors::InvalidEventError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
 
       options[:lifecycle_events][event] << block
     end
 
+    # Checks if running as a server (CLI mode)
+    #
+    # @return [Boolean] true if Shoryuken::CLI is defined
     def server?
       defined?(Shoryuken::CLI)
     end
 
+    # Checks if visibility timeout caching is enabled
+    #
+    # @return [Boolean] true if caching is enabled
     def cache_visibility_timeout?
       @cache_visibility_timeout
     end
 
+    # Checks if ActiveJob queue name prefixing is enabled
+    #
+    # @return [Boolean] true if prefixing is enabled
     def active_job_queue_name_prefixing?
       @active_job_queue_name_prefixing
     end
 
     private
 
+    # Creates the default server middleware chain
+    #
+    # @return [Shoryuken::Middleware::Chain] the default middleware chain
     def default_server_middleware
       Middleware::Chain.new do |m|
         m.add Middleware::Server::Timing
+        m.add Middleware::Server::NonRetryableException
         m.add Middleware::Server::ExponentialBackoffRetry
         m.add Middleware::Server::AutoDelete
         m.add Middleware::Server::AutoExtendVisibility
@@ -183,6 +321,9 @@ module Shoryuken
       end
     end
 
+    # Creates the default client middleware chain
+    #
+    # @return [Shoryuken::Middleware::Chain] an empty middleware chain
     def default_client_middleware
       Middleware::Chain.new
     end