shoryuken 7.0.0.alpha1 → 7.0.0.rc1

data/lib/shoryuken/message.rb

@@ -1,7 +1,70 @@
+# frozen_string_literal: true
+
 module Shoryuken
+  # Represents an SQS message received by a Shoryuken worker.
+  # This class wraps the raw AWS SQS message data and provides convenient methods
+  # for interacting with the message, including deletion and visibility timeout management.
+  #
+  # Message instances are automatically created by Shoryuken and passed to your
+  # worker's `perform` method as the first argument.
+  #
+  # @example Basic worker with message handling
+  #   class MyWorker
+  #     include Shoryuken::Worker
+  #     shoryuken_options queue: 'my_queue'
+  #
+  #     def perform(sqs_msg, body)
+  #       puts "Processing message #{sqs_msg.message_id}"
+  #       puts "Message body: #{body}"
+  #       puts "Queue: #{sqs_msg.queue_name}"
+  #
+  #       # Process the message...
+  #
+  #       # Delete the message when done (if auto_delete is false)
+  #       sqs_msg.delete unless auto_delete?
+  #     end
+  #   end
+  #
+  # @example Working with message attributes
+  #   def perform(sqs_msg, body)
+  #     # Access standard SQS attributes
+  #     sender_id = sqs_msg.attributes['SenderId']
+  #     sent_timestamp = sqs_msg.attributes['SentTimestamp']
+  #
+  #     # Access custom message attributes
+  #     priority = sqs_msg.message_attributes['Priority']&.[]('StringValue')
+  #     user_id = sqs_msg.message_attributes['UserId']&.[]('StringValue')
+  #   end
   class Message
     extend Forwardable
 
+    # @!method message_id
+    #   Returns the unique SQS message ID.
+    #   @return [String] The message ID assigned by SQS
+    #
+    # @!method receipt_handle
+    #   Returns the receipt handle needed for deleting or modifying the message.
+    #   @return [String] The receipt handle for this message
+    #
+    # @!method md5_of_body
+    #   Returns the MD5 hash of the message body.
+    #   @return [String] MD5 hash of the message body
+    #
+    # @!method body
+    #   Returns the raw message body as received from SQS.
+    #   @return [String] The raw message body
+    #
+    # @!method attributes
+    #   Returns the SQS message attributes (system attributes).
+    #   @return [Hash] System attributes like SenderId, SentTimestamp, etc.
+    #
+    # @!method md5_of_message_attributes
+    #   Returns the MD5 hash of the message attributes.
+    #   @return [String] MD5 hash of message attributes
+    #
+    # @!method message_attributes
+    #   Returns custom message attributes set by the sender.
+    #   @return [Hash] Custom message attributes with typed values
     def_delegators(:data,
                    :message_id,
                    :receipt_handle,
@@ -11,8 +74,24 @@ module Shoryuken
                    :md5_of_message_attributes,
                    :message_attributes)
 
-    attr_accessor :client, :queue_url, :queue_name, :data
+    # @return [Aws::SQS::Client] The SQS client used for message operations
+    attr_accessor :client
+
+    # @return [String] The URL of the SQS queue this message came from
+    attr_accessor :queue_url
+
+    # @return [String] The name of the queue this message came from
+    attr_accessor :queue_name
+
+    # @return [Aws::SQS::Types::Message] The raw SQS message data
+    attr_accessor :data
 
+    # Creates a new Message instance wrapping SQS message data.
+    #
+    # @param client [Aws::SQS::Client] The SQS client for message operations
+    # @param queue [Shoryuken::Queue] The queue this message came from
+    # @param data [Aws::SQS::Types::Message] The raw SQS message data
+    # @api private
     def initialize(client, queue, data)
       self.client     = client
       self.data       = data
@@ -20,6 +99,12 @@ module Shoryuken
       self.queue_name = queue.name
     end
 
+    # Deletes this message from the SQS queue.
+    # Once deleted, the message will not be redelivered and cannot be retrieved again.
+    # This is typically called after successful message processing when auto_delete is disabled.
+    #
+    # @return [Aws::SQS::Types::DeleteMessageResult] The deletion result
+    # @raise [Aws::SQS::Errors::ServiceError] If the deletion fails
     def delete
       client.delete_message(
         queue_url: queue_url,
@@ -27,12 +112,42 @@ module Shoryuken
       )
     end
 
+    # Changes the visibility timeout of this message with additional options.
+    # This allows you to hide the message from other consumers for a longer or shorter period.
+    #
+    # @param options [Hash] Options to pass to change_message_visibility
+    # @option options [Integer] :visibility_timeout New visibility timeout in seconds
+    # @return [Aws::SQS::Types::ChangeMessageVisibilityResult] The change result
+    # @raise [Aws::SQS::Errors::ServiceError] If the change fails
+    #
+    # @example Extending visibility with additional options
+    #   sqs_msg.change_visibility(visibility_timeout: 300)
+    #
+    # @see #visibility_timeout= For a simpler interface
     def change_visibility(options)
       client.change_message_visibility(
         options.merge(queue_url: queue_url, receipt_handle: data.receipt_handle)
       )
     end
 
+    # Sets the visibility timeout for this message.
+    # This is a convenience method for changing only the visibility timeout.
+    #
+    # @param timeout [Integer] New visibility timeout in seconds (0-43200)
+    # @return [Aws::SQS::Types::ChangeMessageVisibilityResult] The change result
+    # @raise [Aws::SQS::Errors::ServiceError] If the change fails
+    #
+    # @example Extending processing time
+    #   def perform(sqs_msg, body)
+    #     if complex_processing_needed?(body)
+    #       sqs_msg.visibility_timeout = 1800  # 30 minutes
+    #     end
+    #
+    #     process_message(body)
+    #   end
+    #
+    # @example Making message immediately visible again
+    #   sqs_msg.visibility_timeout = 0  # Make visible immediately
     def visibility_timeout=(timeout)
       client.change_message_visibility(
         queue_url: queue_url,

data/lib/shoryuken/middleware/chain.rb

@@ -1,70 +1,156 @@
+# frozen_string_literal: true
+
 module Shoryuken
-  # Middleware is code configured to run before/after
-  # a message is processed.  It is patterned after Rack
-  # middleware. Middleware exists for the server
-  # side (when jobs are actually processed).
-  #
-  # To modify middleware for the server, just call
-  # with another block:
-  #
-  # Shoryuken.configure_server do |config|
-  #   config.server_middleware do |chain|
-  #     chain.add MyServerHook
-  #     chain.remove ActiveRecord
+  # Middleware provides a way to wrap message processing with custom logic,
+  # similar to Rack middleware in web applications. Middleware runs on the server
+  # side and can perform setup, teardown, error handling, and monitoring around
+  # job execution.
+  #
+  # Middleware classes must implement a `call` method that accepts the worker instance,
+  # queue name, and SQS message, and must yield to continue the middleware chain.
+  #
+  # ## Global Middleware Configuration
+  #
+  # Configure middleware globally for all workers:
+  #
+  #   Shoryuken.configure_server do |config|
+  #     config.server_middleware do |chain|
+  #       chain.add MyServerHook
+  #       chain.remove Shoryuken::Middleware::Server::ActiveRecord
+  #     end
   #   end
-  # end
   #
-  # To insert immediately preceding another entry:
+  # ## Per-Worker Middleware Configuration
   #
-  # Shoryuken.configure_server do |config|
-  #   config.server_middleware do |chain|
-  #     chain.insert_before ActiveRecord, MyServerHook
+  # Configure middleware for specific workers:
+  #
+  #   class MyWorker
+  #     include Shoryuken::Worker
+  #
+  #     server_middleware do |chain|
+  #       chain.add MyWorkerSpecificMiddleware
+  #     end
   #   end
-  # end
   #
-  # To insert immediately after another entry:
+  # ## Middleware Ordering
+  #
+  # Insert middleware at specific positions in the chain:
+  #
+  #   # Insert before existing middleware
+  #   chain.insert_before Shoryuken::Middleware::Server::ActiveRecord, MyDatabaseSetup
+  #
+  #   # Insert after existing middleware
+  #   chain.insert_after Shoryuken::Middleware::Server::Timing, MyMetricsCollector
   #
-  # Shoryuken.configure_server do |config|
-  #   config.server_middleware do |chain|
-  #     chain.insert_after ActiveRecord, MyServerHook
+  #   # Add to beginning of chain
+  #   chain.prepend MyFirstMiddleware
+  #
+  # ## Example Middleware Implementations
+  #
+  #   # Basic logging middleware
+  #   class LoggingMiddleware
+  #     def call(worker_instance, queue, sqs_msg, body)
+  #       puts "Processing #{sqs_msg.message_id} on #{queue}"
+  #       start_time = Time.now
+  #       yield
+  #       puts "Completed in #{Time.now - start_time}s"
+  #     end
   #   end
-  # end
   #
-  # This is an example of a minimal server middleware:
+  #   # Error reporting middleware
+  #   class ErrorReportingMiddleware
+  #     def call(worker_instance, queue, sqs_msg, body)
+  #       yield
+  #     rescue => error
+  #       ErrorReporter.notify(error, {
+  #         worker: worker_instance.class.name,
+  #         queue: queue,
+  #         message_id: sqs_msg.message_id
+  #       })
+  #       raise
+  #     end
+  #   end
   #
-  # class MyServerHook
-  #   def call(worker_instance, queue, sqs_msg)
-  #     puts 'Before work'
-  #     yield
-  #     puts 'After work'
+  #   # Performance monitoring middleware
+  #   class MetricsMiddleware
+  #     def call(worker_instance, queue, sqs_msg, body)
+  #       start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  #       yield
+  #       duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
+  #       StatsD.timing("shoryuken.#{worker_instance.class.name.underscore}.duration", duration)
+  #     end
   #   end
-  # end
   #
+  # @see Shoryuken::Middleware::Chain Middleware chain management
+  # @see https://github.com/ruby-shoryuken/shoryuken/wiki/Middleware Comprehensive middleware guide
   module Middleware
+    # Manages a chain of middleware classes that will be instantiated and invoked
+    # in sequence around message processing. Provides methods for adding, removing,
+    # and reordering middleware.
     class Chain
+      # @return [Array<Entry>] The ordered list of middleware entries
       attr_reader :entries
 
+      # Creates a new middleware chain.
+      #
+      # @yield [Chain] The chain instance for configuration
+      # @example Creating and configuring a chain
+      #   chain = Shoryuken::Middleware::Chain.new do |c|
+      #     c.add MyMiddleware
+      #     c.add AnotherMiddleware, option: 'value'
+      #   end
       def initialize
         @entries = []
         yield self if block_given?
       end
 
+      # Creates a copy of this middleware chain.
+      #
+      # @return [Chain] A new chain with the same middleware entries
       def dup
         self.class.new.tap { |new_chain| new_chain.entries.replace(entries) }
       end
 
+      # Removes all instances of the specified middleware class from the chain.
+      #
+      # @param klass [Class] The middleware class to remove
+      # @return [Array<Entry>] The removed entries
+      # @example Removing ActiveRecord middleware
+      #   chain.remove Shoryuken::Middleware::Server::ActiveRecord
       def remove(klass)
         entries.delete_if { |entry| entry.klass == klass }
       end
 
+      # Adds middleware to the end of the chain. Does nothing if the middleware
+      # class is already present in the chain.
+      #
+      # @param klass [Class] The middleware class to add
+      # @param args [Array] Arguments to pass to the middleware constructor
+      # @example Adding middleware with arguments
+      #   chain.add MyMiddleware, timeout: 30, retries: 3
       def add(klass, *args)
         entries << Entry.new(klass, *args) unless exists?(klass)
       end
 
+      # Adds middleware to the beginning of the chain. Does nothing if the middleware
+      # class is already present in the chain.
+      #
+      # @param klass [Class] The middleware class to prepend
+      # @param args [Array] Arguments to pass to the middleware constructor
+      # @example Adding middleware to run first
+      #   chain.prepend AuthenticationMiddleware
       def prepend(klass, *args)
         entries.insert(0, Entry.new(klass, *args)) unless exists?(klass)
       end
 
+      # Inserts middleware immediately before another middleware class.
+      # If the new middleware already exists, it's moved to the new position.
+      #
+      # @param oldklass [Class] The existing middleware to insert before
+      # @param newklass [Class] The middleware class to insert
+      # @param args [Array] Arguments to pass to the middleware constructor
+      # @example Insert database setup before ActiveRecord middleware
+      #   chain.insert_before Shoryuken::Middleware::Server::ActiveRecord, DatabaseSetup
       def insert_before(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
@@ -72,6 +158,14 @@ module Shoryuken
         entries.insert(i, new_entry)
       end
 
+      # Inserts middleware immediately after another middleware class.
+      # If the new middleware already exists, it's moved to the new position.
+      #
+      # @param oldklass [Class] The existing middleware to insert after
+      # @param newklass [Class] The middleware class to insert
+      # @param args [Array] Arguments to pass to the middleware constructor
+      # @example Insert metrics collection after timing middleware
+      #   chain.insert_after Shoryuken::Middleware::Server::Timing, MetricsCollector
       def insert_after(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
@@ -79,18 +173,34 @@ module Shoryuken
         entries.insert(i + 1, new_entry)
       end
 
+      # Checks if a middleware class is already in the chain.
+      #
+      # @param klass [Class] The middleware class to check for
+      # @return [Boolean] True if the middleware is in the chain
       def exists?(klass)
         entries.any? { |entry| entry.klass == klass }
       end
 
+      # Creates instances of all middleware classes in the chain.
+      #
+      # @return [Array] Array of middleware instances
       def retrieve
         entries.map(&:make_new)
       end
 
+      # Removes all middleware from the chain.
+      #
+      # @return [Array] Empty array
       def clear
         entries.clear
       end
 
+      # Invokes the middleware chain with the given arguments.
+      # Each middleware's call method will be invoked in sequence,
+      # with control passed through yielding.
+      #
+      # @param args [Array] Arguments to pass to each middleware
+      # @yield The final action to perform after all middleware
       def invoke(*args, &final_action)
         chain = retrieve.dup
         traverse_chain = lambda do
@@ -103,18 +213,5 @@ module Shoryuken
         traverse_chain.call
       end
     end
-
-    class Entry
-      attr_reader :klass
-
-      def initialize(klass, *args)
-        @klass = klass
-        @args  = args
-      end
-
-      def make_new
-        @klass.new(*@args)
-      end
-    end
   end
 end

data/lib/shoryuken/middleware/entry.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Middleware
+    # Represents an entry in a middleware chain, storing the middleware class
+    # and any arguments needed for its instantiation.
+    #
+    # @api private
+    class Entry
+      # @return [Class] The middleware class this entry represents
+      attr_reader :klass
+
+      # Creates a new middleware entry.
+      #
+      # @param klass [Class] The middleware class
+      # @param args [Array] Arguments to pass to the middleware constructor
+      def initialize(klass, *args)
+        @klass = klass
+        @args  = args
+      end
+
+      # Creates a new instance of the middleware class with the stored arguments.
+      #
+      # @return [Object] A new instance of the middleware class
+      def make_new
+        @klass.new(*@args)
+      end
+    end
+  end
+end

data/lib/shoryuken/middleware/server/active_record.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server

data/lib/shoryuken/middleware/server/auto_delete.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server

data/lib/shoryuken/middleware/server/auto_extend_visibility.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server
@@ -28,17 +30,15 @@ module Shoryuken
           def auto_extend(_worker, queue, sqs_msg, _body)
             queue_visibility_timeout = Shoryuken::Client.queues(queue).visibility_timeout
 
-            Concurrent::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
-              begin
-                logger.debug do
-                  "Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
-                end
-
-                sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
-              rescue => ex
-                logger.error do
-                  "Could not auto extend the message #{queue}/#{sqs_msg.message_id} visibility timeout. Error: #{ex.message}"
-                end
+            Shoryuken::Helpers::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
+              logger.debug do
+                "Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
+              end
+
+              sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
+            rescue => e
+              logger.error do
+                "Could not auto extend the message #{queue}/#{sqs_msg.message_id} visibility timeout. Error: #{e.message}"
               end
             end
           end

data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server
@@ -14,7 +16,7 @@ module Shoryuken
 
           started_at = Time.now
           yield
-        rescue => ex
+        rescue => e
           retry_intervals = worker.class.get_shoryuken_options['retry_intervals']
 
           if retry_intervals.nil? || !handle_failure(sqs_msg, started_at, retry_intervals)
@@ -23,9 +25,9 @@ module Shoryuken
             raise
           end
 
-          logger.warn { "Message #{sqs_msg.message_id} will attempt retry due to error: #{ex.message}" }
+          logger.warn { "Message #{sqs_msg.message_id} will attempt retry due to error: #{e.message}" }
           # since we didn't raise, lets log the backtrace for debugging purposes.
-          logger.debug { ex.backtrace.join("\n") } unless ex.backtrace.nil?
+          logger.debug { e.backtrace.join("\n") } unless e.backtrace.nil?
         end
 
         private

data/lib/shoryuken/middleware/server/timing.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server

data/lib/shoryuken/options.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class Options
     DEFAULTS = {
@@ -17,10 +19,12 @@ module Shoryuken
       }
     }.freeze
 
-    attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout, :groups,
-                  :launcher_executor, :reloader, :enable_reloading,
-                  :start_callback, :stop_callback, :worker_executor, :worker_registry, :exception_handlers
-    attr_writer :default_worker_options, :sqs_client
+    attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout,
+                  :groups, :launcher_executor, :reloader, :enable_reloading,
+                  :start_callback, :stop_callback, :worker_executor, :worker_registry,
+                  :exception_handlers
+
+    attr_writer :default_worker_options, :sqs_client, :logger
     attr_reader :sqs_client_receive_message_opts
 
     def initialize
@@ -96,7 +100,7 @@ module Shoryuken
     end
 
     def logger
-      Shoryuken::Logging.logger
+      @logger ||= Shoryuken::Logging.logger
     end
 
     def thread_priority

data/lib/shoryuken/polling/base_strategy.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Polling
+    # Abstract base class for queue polling strategies.
+    #
+    # This class defines the interface that all polling strategies must implement
+    # to manage queue selection and message flow control in Shoryuken workers.
+    # Polling strategies determine which queue to fetch messages from next and
+    # how to handle scenarios where queues have no messages available.
+    #
+    # @abstract Subclass and override {#next_queue}, {#messages_found}, and {#active_queues}
+    #   to implement a custom polling strategy.
+    #
+    # @example Implementing a custom polling strategy
+    #   class CustomStrategy < BaseStrategy
+    #     def initialize(queues)
+    #       @queues = queues
+    #     end
+    #
+    #     def next_queue
+    #       # Return next queue to poll
+    #       @queues.sample
+    #     end
+    #
+    #     def messages_found(queue, count)
+    #       # Handle result of polling
+    #       logger.info "Found #{count} messages in #{queue}"
+    #     end
+    #
+    #     def active_queues
+    #       # Return list of active queues
+    #       @queues
+    #     end
+    #   end
+    #
+    # @see WeightedRoundRobin
+    # @see StrictPriority
+    class BaseStrategy
+      include Util
+
+      # Returns the next queue to poll for messages.
+      #
+      # This method should return a QueueConfiguration object representing
+      # the next queue that should be polled for messages, or nil if no
+      # queues are currently available for polling.
+      #
+      # @abstract Subclasses must implement this method
+      # @return [QueueConfiguration, nil] Next queue to poll, or nil if none available
+      # @raise [NotImplementedError] if not implemented by subclass
+      def next_queue
+        fail NotImplementedError
+      end
+
+      # Called when messages are found (or not found) in a queue.
+      #
+      # This method is invoked after polling a queue to inform the strategy
+      # about the number of messages that were retrieved. Strategies can use
+      # this information to make decisions about future polling behavior,
+      # such as pausing empty queues or adjusting queue weights.
+      #
+      # @abstract Subclasses must implement this method
+      # @param _queue [String] The name of the queue that was polled
+      # @param _messages_found [Integer] The number of messages retrieved
+      # @raise [NotImplementedError] if not implemented by subclass
+      def messages_found(_queue, _messages_found)
+        fail NotImplementedError
+      end
+
+      # Called when a message from a queue has been processed.
+      #
+      # This optional callback is invoked after a message has been successfully
+      # processed by a worker. Strategies can use this information for cleanup
+      # or to adjust their polling behavior.
+      #
+      # @param _queue [String] The name of the queue whose message was processed
+      # @return [void]
+      def message_processed(_queue); end
+
+      # Returns the list of currently active queues.
+      #
+      # This method should return an array representing the queues that are
+      # currently active and available for polling. The format may vary by
+      # strategy implementation.
+      #
+      # @abstract Subclasses must implement this method
+      # @return [Array] List of active queues
+      # @raise [NotImplementedError] if not implemented by subclass
+      def active_queues
+        fail NotImplementedError
+      end
+
+      # Compares this strategy with another object for equality.
+      #
+      # Two strategies are considered equal if they have the same active queues.
+      # This method also supports comparison with Array objects for backward
+      # compatibility.
+      #
+      # @param other [Object] Object to compare with
+      # @return [Boolean] true if strategies are equivalent
+      def ==(other)
+        case other
+        when Array
+          @queues == other
+        else
+          if other.respond_to?(:active_queues)
+            active_queues == other.active_queues
+          else
+            false
+          end
+        end
+      end
+
+      # Returns the delay time for pausing empty queues.
+      #
+      # This method returns the amount of time (in seconds) that empty queues
+      # should be paused before being polled again. The delay can be set at
+      # the strategy level or falls back to the global Shoryuken delay setting.
+      #
+      # @return [Float] Delay time in seconds
+      def delay
+        @delay || Shoryuken.options[:delay].to_f
+      end
+    end
+  end
+end