shoryuken 7.0.0.alpha2 → 7.0.0.rc1

data/lib/shoryuken/middleware/chain.rb

@@ -1,72 +1,156 @@
 # frozen_string_literal: true
 
 module Shoryuken
-  # Middleware is code configured to run before/after
-  # a message is processed.  It is patterned after Rack
-  # middleware. Middleware exists for the server
-  # side (when jobs are actually processed).
-  #
-  # To modify middleware for the server, just call
-  # with another block:
-  #
-  # Shoryuken.configure_server do |config|
-  #   config.server_middleware do |chain|
-  #     chain.add MyServerHook
-  #     chain.remove ActiveRecord
+  # Middleware provides a way to wrap message processing with custom logic,
+  # similar to Rack middleware in web applications. Middleware runs on the server
+  # side and can perform setup, teardown, error handling, and monitoring around
+  # job execution.
+  #
+  # Middleware classes must implement a `call` method that accepts the worker instance,
+  # queue name, and SQS message, and must yield to continue the middleware chain.
+  #
+  # ## Global Middleware Configuration
+  #
+  # Configure middleware globally for all workers:
+  #
+  #   Shoryuken.configure_server do |config|
+  #     config.server_middleware do |chain|
+  #       chain.add MyServerHook
+  #       chain.remove Shoryuken::Middleware::Server::ActiveRecord
+  #     end
   #   end
-  # end
   #
-  # To insert immediately preceding another entry:
+  # ## Per-Worker Middleware Configuration
+  #
+  # Configure middleware for specific workers:
   #
-  # Shoryuken.configure_server do |config|
-  #   config.server_middleware do |chain|
-  #     chain.insert_before ActiveRecord, MyServerHook
+  #   class MyWorker
+  #     include Shoryuken::Worker
+  #
+  #     server_middleware do |chain|
+  #       chain.add MyWorkerSpecificMiddleware
+  #     end
   #   end
-  # end
   #
-  # To insert immediately after another entry:
+  # ## Middleware Ordering
+  #
+  # Insert middleware at specific positions in the chain:
+  #
+  #   # Insert before existing middleware
+  #   chain.insert_before Shoryuken::Middleware::Server::ActiveRecord, MyDatabaseSetup
+  #
+  #   # Insert after existing middleware
+  #   chain.insert_after Shoryuken::Middleware::Server::Timing, MyMetricsCollector
   #
-  # Shoryuken.configure_server do |config|
-  #   config.server_middleware do |chain|
-  #     chain.insert_after ActiveRecord, MyServerHook
+  #   # Add to beginning of chain
+  #   chain.prepend MyFirstMiddleware
+  #
+  # ## Example Middleware Implementations
+  #
+  #   # Basic logging middleware
+  #   class LoggingMiddleware
+  #     def call(worker_instance, queue, sqs_msg, body)
+  #       puts "Processing #{sqs_msg.message_id} on #{queue}"
+  #       start_time = Time.now
+  #       yield
+  #       puts "Completed in #{Time.now - start_time}s"
+  #     end
   #   end
-  # end
   #
-  # This is an example of a minimal server middleware:
+  #   # Error reporting middleware
+  #   class ErrorReportingMiddleware
+  #     def call(worker_instance, queue, sqs_msg, body)
+  #       yield
+  #     rescue => error
+  #       ErrorReporter.notify(error, {
+  #         worker: worker_instance.class.name,
+  #         queue: queue,
+  #         message_id: sqs_msg.message_id
+  #       })
+  #       raise
+  #     end
+  #   end
   #
-  # class MyServerHook
-  #   def call(worker_instance, queue, sqs_msg)
-  #     puts 'Before work'
-  #     yield
-  #     puts 'After work'
+  #   # Performance monitoring middleware
+  #   class MetricsMiddleware
+  #     def call(worker_instance, queue, sqs_msg, body)
+  #       start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  #       yield
+  #       duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
+  #       StatsD.timing("shoryuken.#{worker_instance.class.name.underscore}.duration", duration)
+  #     end
   #   end
-  # end
   #
+  # @see Shoryuken::Middleware::Chain Middleware chain management
+  # @see https://github.com/ruby-shoryuken/shoryuken/wiki/Middleware Comprehensive middleware guide
   module Middleware
+    # Manages a chain of middleware classes that will be instantiated and invoked
+    # in sequence around message processing. Provides methods for adding, removing,
+    # and reordering middleware.
     class Chain
+      # @return [Array<Entry>] The ordered list of middleware entries
       attr_reader :entries
 
+      # Creates a new middleware chain.
+      #
+      # @yield [Chain] The chain instance for configuration
+      # @example Creating and configuring a chain
+      #   chain = Shoryuken::Middleware::Chain.new do |c|
+      #     c.add MyMiddleware
+      #     c.add AnotherMiddleware, option: 'value'
+      #   end
       def initialize
         @entries = []
         yield self if block_given?
       end
 
+      # Creates a copy of this middleware chain.
+      #
+      # @return [Chain] A new chain with the same middleware entries
       def dup
         self.class.new.tap { |new_chain| new_chain.entries.replace(entries) }
       end
 
+      # Removes all instances of the specified middleware class from the chain.
+      #
+      # @param klass [Class] The middleware class to remove
+      # @return [Array<Entry>] The removed entries
+      # @example Removing ActiveRecord middleware
+      #   chain.remove Shoryuken::Middleware::Server::ActiveRecord
       def remove(klass)
         entries.delete_if { |entry| entry.klass == klass }
       end
 
+      # Adds middleware to the end of the chain. Does nothing if the middleware
+      # class is already present in the chain.
+      #
+      # @param klass [Class] The middleware class to add
+      # @param args [Array] Arguments to pass to the middleware constructor
+      # @example Adding middleware with arguments
+      #   chain.add MyMiddleware, timeout: 30, retries: 3
       def add(klass, *args)
         entries << Entry.new(klass, *args) unless exists?(klass)
       end
 
+      # Adds middleware to the beginning of the chain. Does nothing if the middleware
+      # class is already present in the chain.
+      #
+      # @param klass [Class] The middleware class to prepend
+      # @param args [Array] Arguments to pass to the middleware constructor
+      # @example Adding middleware to run first
+      #   chain.prepend AuthenticationMiddleware
       def prepend(klass, *args)
         entries.insert(0, Entry.new(klass, *args)) unless exists?(klass)
       end
 
+      # Inserts middleware immediately before another middleware class.
+      # If the new middleware already exists, it's moved to the new position.
+      #
+      # @param oldklass [Class] The existing middleware to insert before
+      # @param newklass [Class] The middleware class to insert
+      # @param args [Array] Arguments to pass to the middleware constructor
+      # @example Insert database setup before ActiveRecord middleware
+      #   chain.insert_before Shoryuken::Middleware::Server::ActiveRecord, DatabaseSetup
       def insert_before(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
@@ -74,6 +158,14 @@ module Shoryuken
         entries.insert(i, new_entry)
       end
 
+      # Inserts middleware immediately after another middleware class.
+      # If the new middleware already exists, it's moved to the new position.
+      #
+      # @param oldklass [Class] The existing middleware to insert after
+      # @param newklass [Class] The middleware class to insert
+      # @param args [Array] Arguments to pass to the middleware constructor
+      # @example Insert metrics collection after timing middleware
+      #   chain.insert_after Shoryuken::Middleware::Server::Timing, MetricsCollector
       def insert_after(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
@@ -81,18 +173,34 @@ module Shoryuken
         entries.insert(i + 1, new_entry)
       end
 
+      # Checks if a middleware class is already in the chain.
+      #
+      # @param klass [Class] The middleware class to check for
+      # @return [Boolean] True if the middleware is in the chain
       def exists?(klass)
         entries.any? { |entry| entry.klass == klass }
       end
 
+      # Creates instances of all middleware classes in the chain.
+      #
+      # @return [Array] Array of middleware instances
       def retrieve
         entries.map(&:make_new)
       end
 
+      # Removes all middleware from the chain.
+      #
+      # @return [Array] Empty array
       def clear
         entries.clear
       end
 
+      # Invokes the middleware chain with the given arguments.
+      # Each middleware's call method will be invoked in sequence,
+      # with control passed through yielding.
+      #
+      # @param args [Array] Arguments to pass to each middleware
+      # @yield The final action to perform after all middleware
       def invoke(*args, &final_action)
         chain = retrieve.dup
         traverse_chain = lambda do
@@ -105,18 +213,5 @@ module Shoryuken
         traverse_chain.call
       end
     end
-
-    class Entry
-      attr_reader :klass
-
-      def initialize(klass, *args)
-        @klass = klass
-        @args  = args
-      end
-
-      def make_new
-        @klass.new(*@args)
-      end
-    end
   end
 end

data/lib/shoryuken/middleware/entry.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Middleware
+    # Represents an entry in a middleware chain, storing the middleware class
+    # and any arguments needed for its instantiation.
+    #
+    # @api private
+    class Entry
+      # @return [Class] The middleware class this entry represents
+      attr_reader :klass
+
+      # Creates a new middleware entry.
+      #
+      # @param klass [Class] The middleware class
+      # @param args [Array] Arguments to pass to the middleware constructor
+      def initialize(klass, *args)
+        @klass = klass
+        @args  = args
+      end
+
+      # Creates a new instance of the middleware class with the stored arguments.
+      #
+      # @return [Object] A new instance of the middleware class
+      def make_new
+        @klass.new(*@args)
+      end
+    end
+  end
+end

data/lib/shoryuken/middleware/server/auto_extend_visibility.rb

@@ -30,7 +30,7 @@ module Shoryuken
           def auto_extend(_worker, queue, sqs_msg, _body)
             queue_visibility_timeout = Shoryuken::Client.queues(queue).visibility_timeout
 
-            Concurrent::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
+            Shoryuken::Helpers::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
               logger.debug do
                 "Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
               end

data/lib/shoryuken/runner.rb

@@ -16,6 +16,9 @@ module Shoryuken
     include Util
     include Singleton
 
+    # @return [Shoryuken::Launcher, nil] the launcher instance, or nil if not yet initialized
+    attr_reader :launcher
+
     def run(options)
       self_read, self_write = IO.pipe
 

data/lib/shoryuken/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Shoryuken
-  VERSION = '7.0.0.alpha2'
+  VERSION = '7.0.0.rc1'
 end

data/lib/shoryuken/worker.rb

@@ -1,6 +1,46 @@
 # frozen_string_literal: true
 
 module Shoryuken
+  # Worker module provides the core functionality for creating Shoryuken workers
+  # that process messages from Amazon SQS queues.
+  #
+  # Including this module in a class provides methods for configuring queue processing,
+  # enqueueing jobs, and setting up middleware. Workers can be configured for different
+  # processing patterns including single message processing, batch processing, and
+  # various retry and visibility timeout strategies.
+  #
+  # @example Basic worker implementation
+  #   class EmailWorker
+  #     include Shoryuken::Worker
+  #     shoryuken_options queue: 'emails'
+  #
+  #     def perform(sqs_msg, body)
+  #       send_email(body['recipient'], body['subject'], body['content'])
+  #     end
+  #   end
+  #
+  # @example Advanced worker with all options
+  #   class AdvancedWorker
+  #     include Shoryuken::Worker
+  #
+  #     shoryuken_options queue: 'advanced_queue',
+  #                       batch: false,
+  #                       auto_delete: true,
+  #                       auto_visibility_timeout: true,
+  #                       retry_intervals: [1, 5, 25, 125, 625]
+  #
+  #     server_middleware do |chain|
+  #       chain.add MyCustomMiddleware
+  #     end
+  #
+  #     def perform(sqs_msg, body)
+  #       # Worker implementation
+  #     end
+  #   end
+  #
+  # @see ClassMethods#shoryuken_options Primary configuration method
+  # @see ClassMethods#perform_async For enqueueing jobs
+  # @see https://github.com/ruby-shoryuken/shoryuken/wiki/Workers Comprehensive worker documentation
   module Worker
     def self.included(base)
       base.extend(ClassMethods)
@@ -8,35 +48,166 @@ module Shoryuken
     end
 
     module ClassMethods
+      # Enqueues a job to be processed asynchronously by a Shoryuken worker.
+      #
+      # @param body [Object] The job payload that will be passed to the worker's perform method
+      # @param options [Hash] Additional options for job enqueueing
+      # @option options [String] :message_group_id FIFO queue group ID for message ordering
+      # @option options [String] :message_deduplication_id FIFO queue deduplication ID
+      # @option options [Hash] :message_attributes Custom SQS message attributes
+      # @return [String] The message ID of the enqueued job
+      #
+      # @example Basic job enqueueing
+      #   MyWorker.perform_async({ user_id: 123, action: 'send_email' })
+      #
+      # @example FIFO queue with ordering
+      #   MyWorker.perform_async(data, message_group_id: 'user_123')
       def perform_async(body, options = {})
         Shoryuken.worker_executor.perform_async(self, body, options)
       end
 
+      # Enqueues a job to be processed after a specified time interval.
+      #
+      # @param interval [Integer, ActiveSupport::Duration] Delay in seconds, or duration object
+      # @param body [Object] The job payload that will be passed to the worker's perform method
+      # @param options [Hash] Additional options for job enqueueing (see {#perform_async})
+      # @return [String] The message ID of the enqueued job
+      #
+      # @example Delay job by 5 minutes
+      #   MyWorker.perform_in(5.minutes, { user_id: 123 })
+      #
+      # @example Delay job by specific number of seconds
+      #   MyWorker.perform_in(300, { user_id: 123 })
       def perform_in(interval, body, options = {})
         Shoryuken.worker_executor.perform_in(self, interval, body, options)
       end
 
       alias_method :perform_at, :perform_in
 
+      # Configures server-side middleware chain for this worker class.
+      # Middleware runs before and after job processing, similar to Rack middleware.
+      #
+      # @yield [Shoryuken::Middleware::Chain] The middleware chain for configuration
+      # @return [Shoryuken::Middleware::Chain] The configured middleware chain
+      #
+      # @example Adding custom middleware
+      #   class MyWorker
+      #     include Shoryuken::Worker
+      #
+      #     server_middleware do |chain|
+      #       chain.add MyCustomMiddleware
+      #       chain.remove Shoryuken::Middleware::Server::ActiveRecord
+      #     end
+      #   end
       def server_middleware
         @_server_chain ||= Shoryuken.server_middleware.dup
         yield @_server_chain if block_given?
         @_server_chain
       end
 
+      # Configures worker options including queue assignment, processing behavior,
+      # and SQS-specific settings. This is the main configuration method for workers.
+      #
+      # @param opts [Hash] Configuration options for the worker
+      # @option opts [String, Array<String>] :queue Queue name(s) this worker processes
+      # @option opts [Boolean] :batch (false) Process messages in batches of up to 10
+      # @option opts [Boolean] :auto_delete (false) Automatically delete messages after processing
+      # @option opts [Boolean] :auto_visibility_timeout (false) Automatically extend message visibility
+      # @option opts [Array<Integer>] :retry_intervals Exponential backoff retry intervals in seconds
+      # @option opts [Hash] :sqs Additional SQS client options
+      #
+      # @example Basic worker configuration
+      #   class MyWorker
+      #     include Shoryuken::Worker
+      #     shoryuken_options queue: 'my_queue'
+      #
+      #     def perform(sqs_msg, body)
+      #       # Process the message
+      #     end
+      #   end
+      #
+      # @example Worker with auto-delete and retries
+      #   class ReliableWorker
+      #     include Shoryuken::Worker
+      #     shoryuken_options queue: 'important_queue',
+      #                       auto_delete: true,
+      #                       retry_intervals: [1, 5, 25, 125]
+      #   end
+      #
+      # @example Batch processing worker
+      #   class BatchWorker
+      #     include Shoryuken::Worker
+      #     shoryuken_options queue: 'batch_queue', batch: true
+      #
+      #     def perform(sqs_msgs, bodies)
+      #       # Process array of up to 10 messages
+      #       bodies.each { |body| process_item(body) }
+      #     end
+      #   end
+      #
+      # @example Multiple queues with priorities
+      #   class MultiQueueWorker
+      #     include Shoryuken::Worker
+      #     shoryuken_options queue: ['high_priority', 'low_priority']
+      #   end
+      #
+      # @example Auto-extending visibility timeout for long-running jobs
+      #   class LongRunningWorker
+      #     include Shoryuken::Worker
+      #     shoryuken_options queue: 'slow_queue',
+      #                       auto_visibility_timeout: true
+      #
+      #     def perform(sqs_msg, body)
+      #       # Long processing that might exceed visibility timeout
+      #       complex_processing(body)
+      #     end
+      #   end
       def shoryuken_options(opts = {})
         self.shoryuken_options_hash = get_shoryuken_options.merge(stringify_keys(opts || {}))
         normalize_worker_queue!
       end
 
+      # Checks if automatic visibility timeout extension is enabled for this worker.
+      # When enabled, Shoryuken automatically extends the message visibility timeout
+      # during processing to prevent the message from becoming visible to other consumers.
+      #
+      # @return [Boolean] true if auto visibility timeout is enabled
+      #
+      # @see #shoryuken_options Documentation for enabling auto_visibility_timeout
       def auto_visibility_timeout?
         !!get_shoryuken_options['auto_visibility_timeout']
       end
 
+      # Checks if exponential backoff retry is configured for this worker.
+      # When retry intervals are specified, failed jobs will be retried with
+      # increasing delays between attempts.
+      #
+      # @return [Boolean] true if retry intervals are configured
+      #
+      # @example Configuring exponential backoff
+      #   shoryuken_options retry_intervals: [1, 5, 25, 125, 625]
+      #   # Will retry after 1s, 5s, 25s, 125s, then 625s before giving up
+      #
+      # @see #shoryuken_options Documentation for configuring retry_intervals
       def exponential_backoff?
         !!get_shoryuken_options['retry_intervals']
       end
 
+      # Checks if automatic message deletion is enabled for this worker.
+      # When enabled, successfully processed messages are automatically deleted
+      # from the SQS queue. When disabled, you must manually delete messages
+      # or they will become visible again after the visibility timeout.
+      #
+      # @return [Boolean] true if auto delete is enabled
+      #
+      # @example Manual message deletion when auto_delete is false
+      #   def perform(sqs_msg, body)
+      #     process_message(body)
+      #     # Manually delete the message after successful processing
+      #     sqs_msg.delete
+      #   end
+      #
+      # @see #shoryuken_options Documentation for enabling auto_delete
       def auto_delete?
         !!(get_shoryuken_options['delete'] || get_shoryuken_options['auto_delete'])
       end

data/spec/integration/active_job_continuation_spec.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'active_job'
+require 'shoryuken/extensions/active_job_adapter'
+require 'shoryuken/extensions/active_job_extensions'
+
+RSpec.describe 'ActiveJob Continuations Integration' do
+  # Skip all tests in this suite if ActiveJob::Continuable is not available (Rails < 8.0)
+  before(:all) do
+    skip 'ActiveJob::Continuable not available (Rails < 8.0)' unless defined?(ActiveJob::Continuable)
+  end
+
+  # Test job that uses ActiveJob Continuations
+  class ContinuableTestJob < ActiveJob::Base
+    include ActiveJob::Continuable if defined?(ActiveJob::Continuable)
+
+    queue_as :default
+
+    class_attribute :executions_log, default: []
+    class_attribute :checkpoints_reached, default: []
+
+    def perform(max_iterations: 10)
+      self.class.executions_log << { execution: executions, started_at: Time.current }
+
+      step :initialize_work do
+        self.class.checkpoints_reached << "initialize_work_#{executions}"
+      end
+
+      step :process_items, start: cursor || 0 do
+        (cursor..max_iterations).each do |i|
+          self.class.checkpoints_reached << "processing_item_#{i}"
+
+          # Check if we should stop (checkpoint)
+          checkpoint
+
+          # Simulate some work
+          sleep 0.01
+
+          # Advance cursor
+          cursor.advance!
+        end
+      end
+
+      step :finalize_work do
+        self.class.checkpoints_reached << 'finalize_work'
+      end
+
+      self.class.executions_log.last[:completed] = true
+    end
+  end
+
+  describe 'stopping? method (unit tests)' do
+    it 'returns false when launcher is not initialized' do
+      adapter = ActiveJob::QueueAdapters::ShoryukenAdapter.new
+      expect(adapter.stopping?).to be false
+    end
+
+    it 'returns true when launcher is stopping' do
+      launcher = Shoryuken::Launcher.new
+      runner = Shoryuken::Runner.instance
+      runner.instance_variable_set(:@launcher, launcher)
+
+      adapter = ActiveJob::QueueAdapters::ShoryukenAdapter.new
+      expect(adapter.stopping?).to be false
+
+      launcher.instance_variable_set(:@stopping, true)
+      expect(adapter.stopping?).to be true
+    end
+  end
+
+  describe 'timestamp handling for continuation retries' do
+    it 'handles past timestamps for continuation retries' do
+      adapter = ActiveJob::QueueAdapters::ShoryukenAdapter.new
+      job = ContinuableTestJob.new
+      job.sqs_send_message_parameters = {}
+
+      # Mock the queue
+      queue = instance_double(Shoryuken::Queue, fifo?: false)
+      allow(Shoryuken::Client).to receive(:queues).and_return(queue)
+      allow(Shoryuken).to receive(:register_worker)
+      allow(queue).to receive(:send_message) do |params|
+        # Verify past timestamp results in immediate delivery (delay_seconds <= 0)
+        expect(params[:delay_seconds]).to be <= 0
+      end
+
+      # Enqueue with past timestamp (simulating continuation retry)
+      past_timestamp = Time.current.to_f - 60
+      adapter.enqueue_at(job, past_timestamp)
+    end
+  end
+
+  describe 'enqueue_at with continuation timestamps (unit tests)' do
+    let(:adapter) { ActiveJob::QueueAdapters::ShoryukenAdapter.new }
+    let(:job) do
+      job = ContinuableTestJob.new
+      job.sqs_send_message_parameters = {}
+      job
+    end
+    let(:queue) { instance_double(Shoryuken::Queue, fifo?: false) }
+
+    before do
+      allow(Shoryuken::Client).to receive(:queues).and_return(queue)
+      allow(Shoryuken).to receive(:register_worker)
+      @sent_messages = []
+      allow(queue).to receive(:send_message) do |params|
+        @sent_messages << params
+      end
+    end
+
+    it 'accepts past timestamps without error' do
+      past_timestamp = Time.current.to_f - 30
+
+      expect {
+        adapter.enqueue_at(job, past_timestamp)
+      }.not_to raise_error
+
+      expect(@sent_messages.size).to eq(1)
+      expect(@sent_messages.first[:delay_seconds]).to be <= 0
+    end
+
+    it 'accepts current timestamp' do
+      current_timestamp = Time.current.to_f
+
+      expect {
+        adapter.enqueue_at(job, current_timestamp)
+      }.not_to raise_error
+
+      expect(@sent_messages.size).to eq(1)
+      expect(@sent_messages.first[:delay_seconds]).to be_between(-1, 1)
+    end
+
+    it 'accepts future timestamp' do
+      future_timestamp = Time.current.to_f + 30
+
+      expect {
+        adapter.enqueue_at(job, future_timestamp)
+      }.not_to raise_error
+
+      expect(@sent_messages.size).to eq(1)
+      expect(@sent_messages.first[:delay_seconds]).to be > 0
+      expect(@sent_messages.first[:delay_seconds]).to be <= 30
+    end
+  end
+end