shoryuken 7.0.0.alpha2 → 7.0.0

data/lib/shoryuken/logging.rb

@@ -2,40 +2,47 @@
 
 require 'time'
 require 'logger'
+require_relative 'logging/base'
+require_relative 'logging/pretty'
+require_relative 'logging/without_timestamp'
 
 module Shoryuken
+  # Provides logging functionality for Shoryuken.
+  # Manages the global logger instance and fiber-local context.
   module Logging
-    class Base < ::Logger::Formatter
-      def tid
-        Thread.current['shoryuken_tid'] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
-      end
-
-      def context
-        c = Thread.current[:shoryuken_context]
-        c ? " #{c}" : ''
-      end
-    end
-
-    class Pretty < Base
-      # Provide a call() method that returns the formatted message.
-      def call(severity, time, _program_name, message)
-        "#{time.utc.iso8601} #{Process.pid} TID-#{tid}#{context} #{severity}: #{message}\n"
-      end
+    # Executes a block with a fiber-local logging context.
+    # Uses Fiber storage (Ruby 3.2+) for proper isolation in async environments.
+    #
+    # @param msg [String] the context message to set
+    # @yield the block to execute within the context
+    # @return [Object] the result of the block
+    def self.with_context(msg)
+      previous = context_storage[:shoryuken_context]
+      context_storage[:shoryuken_context] = msg
+      yield
+    ensure
+      context_storage[:shoryuken_context] = previous
     end
 
-    class WithoutTimestamp < Base
-      def call(severity, _time, _program_name, message)
-        "pid=#{Process.pid} tid=#{tid}#{context} #{severity}: #{message}\n"
-      end
+    # Returns the current logging context value
+    #
+    # @return [String, nil] the current context or nil if not set
+    def self.current_context
+      context_storage[:shoryuken_context]
     end
 
-    def self.with_context(msg)
-      Thread.current[:shoryuken_context] = msg
-      yield
-    ensure
-      Thread.current[:shoryuken_context] = nil
+    # Returns the Fiber class for fiber-local context storage.
+    # Uses Fiber[] and Fiber[]= (Ruby 3.2+) for proper isolation in async environments.
+    #
+    # @return [Class] the Fiber class
+    def self.context_storage
+      Fiber
     end
 
+    # Initializes a new logger instance
+    #
+    # @param log_target [IO, String] the logging target (file path or IO object)
+    # @return [Logger] the initialized logger
     def self.initialize_logger(log_target = STDOUT)
       @logger = Logger.new(log_target)
       @logger.level = Logger::INFO
@@ -43,10 +50,17 @@ module Shoryuken
       @logger
     end
 
+    # Returns the current logger instance, initializing if needed
+    #
+    # @return [Logger] the logger instance
     def self.logger
       @logger ||= initialize_logger
     end
 
+    # Sets the logger instance
+    #
+    # @param log [Logger, nil] the logger to use, or nil for null logger
+    # @return [Logger] the logger instance
     def self.logger=(log)
       @logger = log || Logger.new('/dev/null')
     end

data/lib/shoryuken/manager.rb

@@ -1,15 +1,28 @@
 # frozen_string_literal: true
 
 module Shoryuken
+  # Manages message dispatching and processing for a single processing group.
+  # Coordinates between the fetcher, polling strategy, and processor.
   class Manager
     include Util
 
+    # Maximum number of messages to fetch in a single batch request
     BATCH_LIMIT = 10
-    # See https://github.com/ruby-shoryuken/shoryuken/issues/348#issuecomment-292847028
+
+    # Minimum interval between dispatch cycles
+    # @see https://github.com/ruby-shoryuken/shoryuken/issues/348#issuecomment-292847028
     MIN_DISPATCH_INTERVAL = 0.1
 
+    # @return [String] the processing group name
     attr_reader :group
 
+    # Initializes a new Manager for a processing group
+    #
+    # @param group [String] the processing group name
+    # @param fetcher [Shoryuken::Fetcher] the message fetcher
+    # @param polling_strategy [Shoryuken::Polling::BaseStrategy] the polling strategy
+    # @param concurrency [Integer] the maximum number of concurrent processors
+    # @param executor [Concurrent::ExecutorService] the executor for async operations
     def initialize(group, fetcher, polling_strategy, concurrency, executor)
       @group                      = group
       @fetcher                    = fetcher
@@ -22,15 +35,24 @@ module Shoryuken
       @dispatching_release_signal = ::Queue.new
     end
 
+    # Starts the dispatch loop
+    #
+    # @return [void]
     def start
       fire_utilization_update_event
       dispatch_loop
     end
 
+    # Signals the manager to stop dispatching new messages
+    #
+    # @return [void]
     def stop_new_dispatching
       @stop_new_dispatching.make_true
     end
 
+    # Waits for any in-progress dispatching to complete
+    #
+    # @return [void]
     def await_dispatching_in_progress
       # There might still be a dispatching on-going, as the response from SQS could take some time
       # We don't want to stop the process before processing incoming messages, as they would stay "in-flight" for some time on SQS
@@ -38,12 +60,18 @@ module Shoryuken
       @dispatching_release_signal.pop
     end
 
+    # Checks if the manager is still running
+    #
+    # @return [Boolean] true if the manager is running
     def running?
       @running.true? && @executor.running?
     end
 
     private
 
+    # The main dispatch loop
+    #
+    # @return [void]
     def dispatch_loop
       if @stop_new_dispatching.true? || !running?
         @dispatching_release_signal << 1
@@ -53,6 +81,9 @@ module Shoryuken
       @executor.post { dispatch }
     end
 
+    # Dispatches messages from a queue
+    #
+    # @return [void]
     def dispatch
       return unless running?
 
@@ -71,14 +102,24 @@ module Shoryuken
       dispatch_loop
     end
 
+    # Returns the count of busy processors
+    #
+    # @return [Integer] the number of busy processors
     def busy
       @busy_processors.value
     end
 
+    # Returns the count of ready processors
+    #
+    # @return [Integer] the number of available processors
     def ready
       @max_processors - busy
     end
 
+    # Handles completion of processor work
+    #
+    # @param queue [String] the queue name
+    # @return [void]
     def processor_done(queue)
       @busy_processors.decrement
       fire_utilization_update_event
@@ -90,6 +131,11 @@ module Shoryuken
       @polling_strategy.message_processed(queue)
     end
 
+    # Assigns a message to a processor
+    #
+    # @param queue_name [String] the queue name
+    # @param sqs_msg [Aws::SQS::Types::Message, Array<Aws::SQS::Types::Message>] the message or batch
+    # @return [Concurrent::Promise, nil] the processing promise or nil if not running
     def assign(queue_name, sqs_msg)
       return unless running?
 
@@ -112,22 +158,38 @@ module Shoryuken
         .rescue { processor_done(queue_name) }
     end
 
+    # Dispatches a batch of messages from a queue
+    #
+    # @param queue [Shoryuken::Polling::QueueConfiguration] the queue configuration
+    # @return [void]
     def dispatch_batch(queue)
       batch = @fetcher.fetch(queue, BATCH_LIMIT)
       @polling_strategy.messages_found(queue.name, batch.size)
       assign(queue.name, patch_batch!(batch)) if batch.any?
     end
 
+    # Dispatches individual messages from a queue
+    #
+    # @param queue [Shoryuken::Polling::QueueConfiguration] the queue configuration
+    # @return [void]
     def dispatch_single_messages(queue)
       messages = @fetcher.fetch(queue, ready)
       @polling_strategy.messages_found(queue.name, messages.size)
       messages.each { |message| assign(queue.name, message) }
     end
 
+    # Checks if a queue uses batch message processing
+    #
+    # @param queue [Shoryuken::Polling::QueueConfiguration] the queue configuration
+    # @return [Boolean] true if the queue is configured for batch processing
     def batched_queue?(queue)
       Shoryuken.worker_registry.batch_receive_messages?(queue.name)
     end
 
+    # Patches a batch array with a message_id method
+    #
+    # @param sqs_msgs [Array<Aws::SQS::Types::Message>] the batch of messages
+    # @return [Array<Aws::SQS::Types::Message>] the patched batch
     def patch_batch!(sqs_msgs)
       sqs_msgs.instance_eval do
         def message_id
@@ -138,6 +200,10 @@ module Shoryuken
       sqs_msgs
     end
 
+    # Handles errors during dispatch
+    #
+    # @param ex [Exception] the exception that occurred
+    # @return [void]
     def handle_dispatch_error(ex)
       logger.error { "Manager failed: #{ex.message}" }
       logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
@@ -147,6 +213,9 @@ module Shoryuken
       @running.make_false
     end
 
+    # Fires a utilization update event
+    #
+    # @return [void]
     def fire_utilization_update_event
       fire_event :utilization_update, false, {
         group: @group,

data/lib/shoryuken/message.rb

@@ -1,9 +1,70 @@
 # frozen_string_literal: true
 
 module Shoryuken
+  # Represents an SQS message received by a Shoryuken worker.
+  # This class wraps the raw AWS SQS message data and provides convenient methods
+  # for interacting with the message, including deletion and visibility timeout management.
+  #
+  # Message instances are automatically created by Shoryuken and passed to your
+  # worker's `perform` method as the first argument.
+  #
+  # @example Basic worker with message handling
+  #   class MyWorker
+  #     include Shoryuken::Worker
+  #     shoryuken_options queue: 'my_queue'
+  #
+  #     def perform(sqs_msg, body)
+  #       puts "Processing message #{sqs_msg.message_id}"
+  #       puts "Message body: #{body}"
+  #       puts "Queue: #{sqs_msg.queue_name}"
+  #
+  #       # Process the message...
+  #
+  #       # Delete the message when done (if auto_delete is false)
+  #       sqs_msg.delete unless auto_delete?
+  #     end
+  #   end
+  #
+  # @example Working with message attributes
+  #   def perform(sqs_msg, body)
+  #     # Access standard SQS attributes
+  #     sender_id = sqs_msg.attributes['SenderId']
+  #     sent_timestamp = sqs_msg.attributes['SentTimestamp']
+  #
+  #     # Access custom message attributes
+  #     priority = sqs_msg.message_attributes['Priority']&.[]('StringValue')
+  #     user_id = sqs_msg.message_attributes['UserId']&.[]('StringValue')
+  #   end
   class Message
     extend Forwardable
 
+    # @!method message_id
+    #   Returns the unique SQS message ID.
+    #   @return [String] The message ID assigned by SQS
+    #
+    # @!method receipt_handle
+    #   Returns the receipt handle needed for deleting or modifying the message.
+    #   @return [String] The receipt handle for this message
+    #
+    # @!method md5_of_body
+    #   Returns the MD5 hash of the message body.
+    #   @return [String] MD5 hash of the message body
+    #
+    # @!method body
+    #   Returns the raw message body as received from SQS.
+    #   @return [String] The raw message body
+    #
+    # @!method attributes
+    #   Returns the SQS message attributes (system attributes).
+    #   @return [Hash] System attributes like SenderId, SentTimestamp, etc.
+    #
+    # @!method md5_of_message_attributes
+    #   Returns the MD5 hash of the message attributes.
+    #   @return [String] MD5 hash of message attributes
+    #
+    # @!method message_attributes
+    #   Returns custom message attributes set by the sender.
+    #   @return [Hash] Custom message attributes with typed values
     def_delegators(:data,
                    :message_id,
                    :receipt_handle,
@@ -13,8 +74,24 @@ module Shoryuken
                    :md5_of_message_attributes,
                    :message_attributes)
 
-    attr_accessor :client, :queue_url, :queue_name, :data
+    # @return [Aws::SQS::Client] The SQS client used for message operations
+    attr_accessor :client
 
+    # @return [String] The URL of the SQS queue this message came from
+    attr_accessor :queue_url
+
+    # @return [String] The name of the queue this message came from
+    attr_accessor :queue_name
+
+    # @return [Aws::SQS::Types::Message] The raw SQS message data
+    attr_accessor :data
+
+    # Creates a new Message instance wrapping SQS message data.
+    #
+    # @param client [Aws::SQS::Client] The SQS client for message operations
+    # @param queue [Shoryuken::Queue] The queue this message came from
+    # @param data [Aws::SQS::Types::Message] The raw SQS message data
+    # @api private
     def initialize(client, queue, data)
       self.client     = client
       self.data       = data
@@ -22,6 +99,12 @@ module Shoryuken
       self.queue_name = queue.name
     end
 
+    # Deletes this message from the SQS queue.
+    # Once deleted, the message will not be redelivered and cannot be retrieved again.
+    # This is typically called after successful message processing when auto_delete is disabled.
+    #
+    # @return [Aws::SQS::Types::DeleteMessageResult] The deletion result
+    # @raise [Aws::SQS::Errors::ServiceError] If the deletion fails
     def delete
       client.delete_message(
         queue_url: queue_url,
@@ -29,12 +112,42 @@ module Shoryuken
       )
     end
 
+    # Changes the visibility timeout of this message with additional options.
+    # This allows you to hide the message from other consumers for a longer or shorter period.
+    #
+    # @param options [Hash] Options to pass to change_message_visibility
+    # @option options [Integer] :visibility_timeout New visibility timeout in seconds
+    # @return [Aws::SQS::Types::ChangeMessageVisibilityResult] The change result
+    # @raise [Aws::SQS::Errors::ServiceError] If the change fails
+    #
+    # @example Extending visibility with additional options
+    #   sqs_msg.change_visibility(visibility_timeout: 300)
+    #
+    # @see #visibility_timeout= For a simpler interface
     def change_visibility(options)
       client.change_message_visibility(
         options.merge(queue_url: queue_url, receipt_handle: data.receipt_handle)
       )
     end
 
+    # Sets the visibility timeout for this message.
+    # This is a convenience method for changing only the visibility timeout.
+    #
+    # @param timeout [Integer] New visibility timeout in seconds (0-43200)
+    # @return [Aws::SQS::Types::ChangeMessageVisibilityResult] The change result
+    # @raise [Aws::SQS::Errors::ServiceError] If the change fails
+    #
+    # @example Extending processing time
+    #   def perform(sqs_msg, body)
+    #     if complex_processing_needed?(body)
+    #       sqs_msg.visibility_timeout = 1800  # 30 minutes
+    #     end
+    #
+    #     process_message(body)
+    #   end
+    #
+    # @example Making message immediately visible again
+    #   sqs_msg.visibility_timeout = 0  # Make visible immediately
     def visibility_timeout=(timeout)
       client.change_message_visibility(
         queue_url: queue_url,

data/lib/shoryuken/middleware/chain.rb

@@ -1,72 +1,156 @@
 # frozen_string_literal: true
 
 module Shoryuken
-  # Middleware is code configured to run before/after
-  # a message is processed.  It is patterned after Rack
-  # middleware. Middleware exists for the server
-  # side (when jobs are actually processed).
-  #
-  # To modify middleware for the server, just call
-  # with another block:
-  #
-  # Shoryuken.configure_server do |config|
-  #   config.server_middleware do |chain|
-  #     chain.add MyServerHook
-  #     chain.remove ActiveRecord
+  # Middleware provides a way to wrap message processing with custom logic,
+  # similar to Rack middleware in web applications. Middleware runs on the server
+  # side and can perform setup, teardown, error handling, and monitoring around
+  # job execution.
+  #
+  # Middleware classes must implement a `call` method that accepts the worker instance,
+  # queue name, and SQS message, and must yield to continue the middleware chain.
+  #
+  # ## Global Middleware Configuration
+  #
+  # Configure middleware globally for all workers:
+  #
+  #   Shoryuken.configure_server do |config|
+  #     config.server_middleware do |chain|
+  #       chain.add MyServerHook
+  #       chain.remove Shoryuken::Middleware::Server::ActiveRecord
+  #     end
   #   end
-  # end
   #
-  # To insert immediately preceding another entry:
+  # ## Per-Worker Middleware Configuration
+  #
+  # Configure middleware for specific workers:
   #
-  # Shoryuken.configure_server do |config|
-  #   config.server_middleware do |chain|
-  #     chain.insert_before ActiveRecord, MyServerHook
+  #   class MyWorker
+  #     include Shoryuken::Worker
+  #
+  #     server_middleware do |chain|
+  #       chain.add MyWorkerSpecificMiddleware
+  #     end
   #   end
-  # end
   #
-  # To insert immediately after another entry:
+  # ## Middleware Ordering
+  #
+  # Insert middleware at specific positions in the chain:
+  #
+  #   # Insert before existing middleware
+  #   chain.insert_before Shoryuken::Middleware::Server::ActiveRecord, MyDatabaseSetup
+  #
+  #   # Insert after existing middleware
+  #   chain.insert_after Shoryuken::Middleware::Server::Timing, MyMetricsCollector
   #
-  # Shoryuken.configure_server do |config|
-  #   config.server_middleware do |chain|
-  #     chain.insert_after ActiveRecord, MyServerHook
+  #   # Add to beginning of chain
+  #   chain.prepend MyFirstMiddleware
+  #
+  # ## Example Middleware Implementations
+  #
+  #   # Basic logging middleware
+  #   class LoggingMiddleware
+  #     def call(worker_instance, queue, sqs_msg, body)
+  #       puts "Processing #{sqs_msg.message_id} on #{queue}"
+  #       start_time = Time.now
+  #       yield
+  #       puts "Completed in #{Time.now - start_time}s"
+  #     end
   #   end
-  # end
   #
-  # This is an example of a minimal server middleware:
+  #   # Error reporting middleware
+  #   class ErrorReportingMiddleware
+  #     def call(worker_instance, queue, sqs_msg, body)
+  #       yield
+  #     rescue => error
+  #       ErrorReporter.notify(error, {
+  #         worker: worker_instance.class.name,
+  #         queue: queue,
+  #         message_id: sqs_msg.message_id
+  #       })
+  #       raise
+  #     end
+  #   end
   #
-  # class MyServerHook
-  #   def call(worker_instance, queue, sqs_msg)
-  #     puts 'Before work'
-  #     yield
-  #     puts 'After work'
+  #   # Performance monitoring middleware
+  #   class MetricsMiddleware
+  #     def call(worker_instance, queue, sqs_msg, body)
+  #       start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  #       yield
+  #       duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
+  #       StatsD.timing("shoryuken.#{worker_instance.class.name.underscore}.duration", duration)
+  #     end
   #   end
-  # end
   #
+  # @see Shoryuken::Middleware::Chain Middleware chain management
+  # @see https://github.com/ruby-shoryuken/shoryuken/wiki/Middleware Comprehensive middleware guide
   module Middleware
+    # Manages a chain of middleware classes that will be instantiated and invoked
+    # in sequence around message processing. Provides methods for adding, removing,
+    # and reordering middleware.
     class Chain
+      # @return [Array<Entry>] The ordered list of middleware entries
       attr_reader :entries
 
+      # Creates a new middleware chain.
+      #
+      # @yield [Chain] The chain instance for configuration
+      # @example Creating and configuring a chain
+      #   chain = Shoryuken::Middleware::Chain.new do |c|
+      #     c.add MyMiddleware
+      #     c.add AnotherMiddleware, option: 'value'
+      #   end
       def initialize
         @entries = []
         yield self if block_given?
       end
 
+      # Creates a copy of this middleware chain.
+      #
+      # @return [Chain] A new chain with the same middleware entries
       def dup
         self.class.new.tap { |new_chain| new_chain.entries.replace(entries) }
       end
 
+      # Removes all instances of the specified middleware class from the chain.
+      #
+      # @param klass [Class] The middleware class to remove
+      # @return [Array<Entry>] The removed entries
+      # @example Removing ActiveRecord middleware
+      #   chain.remove Shoryuken::Middleware::Server::ActiveRecord
       def remove(klass)
         entries.delete_if { |entry| entry.klass == klass }
       end
 
+      # Adds middleware to the end of the chain. Does nothing if the middleware
+      # class is already present in the chain.
+      #
+      # @param klass [Class] The middleware class to add
+      # @param args [Array] Arguments to pass to the middleware constructor
+      # @example Adding middleware with arguments
+      #   chain.add MyMiddleware, timeout: 30, retries: 3
       def add(klass, *args)
         entries << Entry.new(klass, *args) unless exists?(klass)
       end
 
+      # Adds middleware to the beginning of the chain. Does nothing if the middleware
+      # class is already present in the chain.
+      #
+      # @param klass [Class] The middleware class to prepend
+      # @param args [Array] Arguments to pass to the middleware constructor
+      # @example Adding middleware to run first
+      #   chain.prepend AuthenticationMiddleware
       def prepend(klass, *args)
         entries.insert(0, Entry.new(klass, *args)) unless exists?(klass)
       end
 
+      # Inserts middleware immediately before another middleware class.
+      # If the new middleware already exists, it's moved to the new position.
+      #
+      # @param oldklass [Class] The existing middleware to insert before
+      # @param newklass [Class] The middleware class to insert
+      # @param args [Array] Arguments to pass to the middleware constructor
+      # @example Insert database setup before ActiveRecord middleware
+      #   chain.insert_before Shoryuken::Middleware::Server::ActiveRecord, DatabaseSetup
       def insert_before(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
@@ -74,6 +158,14 @@ module Shoryuken
         entries.insert(i, new_entry)
       end
 
+      # Inserts middleware immediately after another middleware class.
+      # If the new middleware already exists, it's moved to the new position.
+      #
+      # @param oldklass [Class] The existing middleware to insert after
+      # @param newklass [Class] The middleware class to insert
+      # @param args [Array] Arguments to pass to the middleware constructor
+      # @example Insert metrics collection after timing middleware
+      #   chain.insert_after Shoryuken::Middleware::Server::Timing, MetricsCollector
       def insert_after(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
@@ -81,18 +173,35 @@ module Shoryuken
         entries.insert(i + 1, new_entry)
       end
 
+      # Checks if a middleware class is already in the chain.
+      #
+      # @param klass [Class] The middleware class to check for
+      # @return [Boolean] True if the middleware is in the chain
       def exists?(klass)
         entries.any? { |entry| entry.klass == klass }
       end
 
+      # Creates instances of all middleware classes in the chain.
+      #
+      # @return [Array] Array of middleware instances
       def retrieve
         entries.map(&:make_new)
       end
 
+      # Removes all middleware from the chain.
+      #
+      # @return [Array] Empty array
       def clear
         entries.clear
       end
 
+      # Invokes the middleware chain with the given arguments.
+      # Each middleware's call method will be invoked in sequence,
+      # with control passed through yielding.
+      #
+      # @param args [Array] arguments to pass to each middleware
+      # @param final_action [Proc] the final action to perform after all middleware
+      # @return [void]
       def invoke(*args, &final_action)
         chain = retrieve.dup
         traverse_chain = lambda do
@@ -105,18 +214,5 @@ module Shoryuken
         traverse_chain.call
       end
     end
-
-    class Entry
-      attr_reader :klass
-
-      def initialize(klass, *args)
-        @klass = klass
-        @args  = args
-      end
-
-      def make_new
-        @klass.new(*@args)
-      end
-    end
   end
 end