shoryuken 7.0.0.alpha2 → 7.0.0

data/lib/active_job/queue_adapters/shoryuken_adapter.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+# ActiveJob docs: http://edgeguides.rubyonrails.org/active_job_basics.html
+# Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
+
+require 'shoryuken'
+require 'shoryuken/active_job/job_wrapper'
+
+# Rails ActiveJob module providing background job processing
+module ActiveJob
+  # Queue adapter implementations for various backends
+  module QueueAdapters
+    # Shoryuken adapter for Active Job.
+    # To use Shoryuken set the queue_adapter config to +:shoryuken+.
+    #
+    # @example Rails configuration
+    #   Rails.application.config.active_job.queue_adapter = :shoryuken
+
+    # Determine the appropriate base class based on Rails version
+    # This prevents AbstractAdapter autoloading issues in Rails 7.0-7.1
+    base = if defined?(Rails) && defined?(Rails::VERSION)
+             (Rails::VERSION::MAJOR == 7 && Rails::VERSION::MINOR < 2 ? Object : AbstractAdapter)
+           else
+             Object
+           end
+
+    # Shoryuken queue adapter for ActiveJob integration.
+    # Provides methods for enqueueing jobs to SQS queues.
+    class ShoryukenAdapter < base
+      class << self
+        # Returns the singleton adapter instance
+        #
+        # @return [ShoryukenAdapter] the adapter instance
+        def instance
+          # https://github.com/ruby-shoryuken/shoryuken/pull/174#issuecomment-174555657
+          @instance ||= new
+        end
+
+        # Enqueues a job for immediate processing
+        #
+        # @param job [ActiveJob::Base] the job to enqueue
+        # @return [Aws::SQS::Types::SendMessageResult] the send result
+        def enqueue(job)
+          instance.enqueue(job)
+        end
+
+        # Enqueues a job for delayed processing
+        #
+        # @param job [ActiveJob::Base] the job to enqueue
+        # @param timestamp [Float] Unix timestamp when the job should be processed
+        # @return [Aws::SQS::Types::SendMessageResult] the send result
+        def enqueue_at(job, timestamp)
+          instance.enqueue_at(job, timestamp)
+        end
+      end
+
+      # Checks if jobs should be enqueued after transaction commit (Rails 7.2+)
+      #
+      # @return [Boolean] always returns true
+      def enqueue_after_transaction_commit?
+        true
+      end
+
+      # Indicates whether Shoryuken is in the process of shutting down.
+      #
+      # This method is required for ActiveJob Continuations support (Rails 8.1+).
+      # When true, it signals to jobs that they should checkpoint their progress
+      # and gracefully interrupt execution to allow for resumption after restart.
+      #
+      # @return [Boolean] true if Shoryuken is shutting down, false otherwise
+      # @see https://github.com/rails/rails/pull/55127 Rails ActiveJob Continuations
+      def stopping?
+        launcher = Shoryuken::Runner.instance.launcher
+        launcher&.stopping? || false
+      end
+
+      # Enqueues a job for immediate processing
+      #
+      # @param job [ActiveJob::Base] the job to enqueue
+      # @param options [Hash] SQS message configuration
+      # @option options [Integer] :delay_seconds delay before the message becomes visible
+      # @option options [String] :message_group_id FIFO queue group ID
+      # @option options [String] :message_deduplication_id FIFO queue deduplication ID
+      # @return [Aws::SQS::Types::SendMessageResult] the send result
+      def enqueue(job, options = {}) # :nodoc:
+        register_worker!(job)
+
+        job.sqs_send_message_parameters.merge! options
+
+        queue = Shoryuken::Client.queues(job.queue_name)
+        send_message_params = message queue, job
+        job.sqs_send_message_parameters = send_message_params
+        queue.send_message send_message_params
+      end
+
+      # Enqueues a job for delayed processing
+      #
+      # @param job [ActiveJob::Base] the job to enqueue
+      # @param timestamp [Float] Unix timestamp when the job should be processed
+      # @return [Aws::SQS::Types::SendMessageResult] the send result
+      # @raise [ArgumentError] if delay is used with a FIFO queue
+      def enqueue_at(job, timestamp) # :nodoc:
+        delay = calculate_delay(timestamp)
+
+        # FIFO queues do not support per-message delays
+        # Check early to fail synchronously (before any async wrapping in subclasses)
+        # Note: negative delays (past timestamps) don't need handling here -
+        # SQS treats them as immediate delivery (delay_seconds=0)
+        # See https://github.com/ruby-shoryuken/shoryuken/issues/924
+        if delay.positive?
+          queue = Shoryuken::Client.queues(job.queue_name)
+          if queue.fifo?
+            raise ArgumentError,
+                  "FIFO queue '#{queue.name}' does not support per-message delays. " \
+                  'When using ActiveJob retry_on with FIFO queues, set `wait: 0`.'
+          end
+        end
+
+        enqueue(job, delay_seconds: delay)
+      end
+
+      # Bulk enqueue multiple jobs efficiently using SQS batch API.
+      # Called by ActiveJob.perform_all_later (Rails 7.1+).
+      #
+      # @param jobs [Array<ActiveJob::Base>] array of ActiveJob instances to be enqueued
+      # @return [Integer] number of jobs successfully enqueued
+      def enqueue_all(jobs) # :nodoc:
+        jobs.group_by(&:queue_name).each do |queue_name, queue_jobs|
+          queue = Shoryuken::Client.queues(queue_name)
+
+          queue_jobs.each_slice(10) do |batch|
+            entries = batch.map.with_index do |job, idx|
+              register_worker!(job)
+              msg = message(queue, job)
+              job.sqs_send_message_parameters = msg
+              { id: idx.to_s }.merge(msg)
+            end
+
+            response = queue.send_messages(entries: entries)
+            successful_ids = response.successful.map { |r| r.id.to_i }.to_set
+            batch.each_with_index do |job, idx|
+              job.successfully_enqueued = successful_ids.include?(idx)
+            end
+          end
+        end
+
+        jobs.count(&:successfully_enqueued?)
+      end
+
+      private
+
+      # Calculates the delay in seconds from a timestamp
+      #
+      # @param timestamp [Float] Unix timestamp
+      # @return [Integer] delay in seconds
+      # @raise [RuntimeError] if delay exceeds 15 minutes
+      def calculate_delay(timestamp)
+        delay = (timestamp - Time.current.to_f).round
+        raise 'The maximum allowed delay is 15 minutes' if delay > 15.minutes
+
+        delay
+      end
+
+      # Builds the SQS message parameters for a job
+      #
+      # @param queue [Shoryuken::Queue] the queue to send to
+      # @param job [ActiveJob::Base] the job to serialize
+      # @return [Hash] the message parameters
+      def message(queue, job)
+        body = job.serialize
+        job_params = job.sqs_send_message_parameters
+
+        attributes = job_params[:message_attributes] || {}
+
+        msg = {
+          message_body: body,
+          message_attributes: attributes.merge(MESSAGE_ATTRIBUTES)
+        }
+
+        if queue.fifo?
+          # See https://github.com/ruby-shoryuken/shoryuken/issues/457 and
+          # https://github.com/ruby-shoryuken/shoryuken/pull/750#issuecomment-1781317929
+          msg[:message_deduplication_id] = Digest::SHA256.hexdigest(
+            JSON.dump(body.except('job_id', 'enqueued_at'))
+          )
+        end
+
+        msg.merge(job_params.except(:message_attributes))
+      end
+
+      # Registers the JobWrapper as the worker for the job's queue
+      #
+      # @param job [ActiveJob::Base] the job being enqueued
+      # @return [void]
+      def register_worker!(job)
+        Shoryuken.register_worker(job.queue_name, Shoryuken::ActiveJob::JobWrapper)
+      end
+
+      # Default message attributes identifying the Shoryuken worker class
+      MESSAGE_ATTRIBUTES = {
+        'shoryuken_class' => {
+          string_value: Shoryuken::ActiveJob::JobWrapper.to_s,
+          data_type: 'String'
+        }
+      }.freeze
+    end
+  end
+end

data/lib/active_job/queue_adapters/shoryuken_concurrent_send_adapter.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# ActiveJob docs: http://edgeguides.rubyonrails.org/active_job_basics.html
+# Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
+require_relative 'shoryuken_adapter'
+
+module ActiveJob
+  module QueueAdapters
+    # Shoryuken concurrent adapter for Active Job.
+    #
+    # This adapter sends messages asynchronously (ie non-blocking) and allows
+    # the caller to set up handlers for both success and failure.
+    #
+    # @example Setting up the adapter
+    #   success_handler = ->(response, job, options) { StatsD.increment("#{job.class.name}.success") }
+    #   error_handler = ->(err, job, options) { StatsD.increment("#{job.class.name}.failure") }
+    #
+    #   adapter = ActiveJob::QueueAdapters::ShoryukenConcurrentSendAdapter.new(success_handler, error_handler)
+    #
+    #   config.active_job.queue_adapter = adapter
+    class ShoryukenConcurrentSendAdapter < ShoryukenAdapter
+      # Initializes a new concurrent send adapter
+      #
+      # @param success_handler [Proc, nil] callback for successful enqueues
+      # @param error_handler [Proc, nil] callback for failed enqueues
+      def initialize(success_handler = nil, error_handler = nil)
+        super() if defined?(super)
+        @success_handler = success_handler
+        @error_handler = error_handler
+      end
+
+      # Enqueues a job asynchronously
+      #
+      # @param job [ActiveJob::Base] the job to enqueue
+      # @param options [Hash] SQS message configuration
+      # @option options [Integer] :delay_seconds delay before the message becomes visible
+      # @option options [String] :message_group_id FIFO queue group ID
+      # @option options [String] :message_deduplication_id FIFO queue deduplication ID
+      # @return [Concurrent::Promises::Future] the future representing the async operation
+      def enqueue(job, options = {})
+        send_concurrently(job, options) { |f_job, f_options| super(f_job, f_options) }
+      end
+
+      # Returns the success handler, using a default no-op if not set
+      #
+      # @return [Proc] the success handler
+      def success_handler
+        @success_handler ||= ->(_send_message_response, _job, _options) { nil }
+      end
+
+      # Returns the error handler, using a default logger if not set
+      #
+      # @return [Proc] the error handler
+      def error_handler
+        @error_handler ||= lambda { |error, job, _options|
+          Shoryuken.logger.warn("Failed to enqueue job: #{job.inspect} due to error: #{error}")
+        }
+      end
+
+      private
+
+      # Sends a message concurrently using futures
+      #
+      # @param job [ActiveJob::Base] the job to enqueue
+      # @param options [Hash] SQS message configuration passed to the enqueue operation
+      # @option options [Integer] :delay_seconds delay before the message becomes visible
+      # @option options [String] :message_group_id FIFO queue group ID
+      # @yield [job, options] the actual enqueue operation
+      # @return [Concurrent::Promises::Future] the future representing the async operation
+      def send_concurrently(job, options)
+        Concurrent::Promises
+          .future(job, options) { |f_job, f_options| [yield(f_job, f_options), f_job, f_options] }
+          .then { |send_message_response, f_job, f_options| success_handler.call(send_message_response, f_job, f_options) }
+          .rescue(job, options) { |err, f_job, f_options| error_handler.call(err, f_job, f_options) }
+      end
+    end
+  end
+end

data/lib/shoryuken/active_job/current_attributes.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require 'active_support/current_attributes'
+require 'active_job'
+
+module Shoryuken
+  # ActiveJob integration module for Shoryuken
+  module ActiveJob
+    # Middleware to persist Rails CurrentAttributes across job execution.
+    #
+    # This ensures that request-scoped context (like current user, tenant, locale)
+    # automatically flows from the code that enqueues a job to the job's execution.
+    #
+    # Based on Sidekiq's approach to persisting current attributes.
+    #
+    # @example Setup in initializer
+    #   require 'shoryuken/active_job/current_attributes'
+    #   Shoryuken::ActiveJob::CurrentAttributes.persist('MyApp::Current')
+    #
+    # @example Multiple CurrentAttributes classes
+    #   Shoryuken::ActiveJob::CurrentAttributes.persist('MyApp::Current', 'MyApp::RequestContext')
+    #
+    # @see https://api.rubyonrails.org/classes/ActiveSupport/CurrentAttributes.html
+    # @see https://github.com/sidekiq/sidekiq/blob/main/lib/sidekiq/middleware/current_attributes.rb
+    module CurrentAttributes
+      # Serializer for current attributes using ActiveJob::Arguments.
+      # Supports Symbols and GlobalID objects.
+      module Serializer
+        module_function
+
+        # Serializes attributes hash for SQS message storage
+        #
+        # @param attrs [Hash] the attributes to serialize
+        # @return [Object] the serialized attributes
+        def serialize(attrs)
+          ::ActiveJob::Arguments.serialize([attrs]).first
+        end
+
+        # Deserializes attributes hash from SQS message
+        #
+        # @param attrs [Object] the serialized attributes
+        # @return [Hash] the deserialized attributes
+        def deserialize(attrs)
+          ::ActiveJob::Arguments.deserialize([attrs]).first
+        end
+      end
+
+      class << self
+        # @return [Hash{String => String}] serialization keys mapped to CurrentAttributes class names
+        attr_reader :cattrs
+
+        # Register CurrentAttributes classes to persist across job execution.
+        #
+        # @param klasses [Array<String, Class>] CurrentAttributes class names or classes
+        # @example
+        #   Shoryuken::ActiveJob::CurrentAttributes.persist('Current')
+        #   Shoryuken::ActiveJob::CurrentAttributes.persist(Current, RequestContext)
+        def persist(*klasses)
+          @cattrs ||= {}
+
+          klasses.flatten.each_with_index do |klass, idx|
+            key = @cattrs.empty? ? 'cattr' : "cattr_#{idx}"
+            @cattrs[key] = klass.to_s
+          end
+
+          # Prepend the persistence module to the adapter for serialization
+          unless ::ActiveJob::QueueAdapters::ShoryukenAdapter.ancestors.include?(Persistence)
+            ::ActiveJob::QueueAdapters::ShoryukenAdapter.prepend(Persistence)
+          end
+
+          # Prepend the loading module to JobWrapper for deserialization
+          unless Shoryuken::ActiveJob::JobWrapper.ancestors.include?(Loading)
+            Shoryuken::ActiveJob::JobWrapper.prepend(Loading)
+          end
+        end
+      end
+
+      # Module prepended to ShoryukenAdapter to serialize CurrentAttributes on enqueue.
+      module Persistence
+        private
+
+        # Builds the SQS message with CurrentAttributes data
+        #
+        # @param queue [Shoryuken::Queue] the target queue
+        # @param job [ActiveJob::Base] the job being enqueued
+        # @return [Hash] the message parameters
+        def message(queue, job)
+          hash = super
+
+          CurrentAttributes.cattrs&.each do |key, klass_name|
+            next if hash[:message_body].key?(key)
+
+            klass = klass_name.constantize
+            attrs = klass.attributes
+            next if attrs.empty?
+
+            hash[:message_body][key] = Serializer.serialize(attrs)
+          end
+
+          hash
+        end
+      end
+
+      # Module prepended to JobWrapper to restore CurrentAttributes on execute.
+      module Loading
+        # Performs the job after restoring CurrentAttributes
+        #
+        # @param sqs_msg [Shoryuken::Message] the SQS message
+        # @param hash [Hash] the deserialized job data
+        # @return [void]
+        def perform(sqs_msg, hash)
+          klasses_to_reset = []
+
+          CurrentAttributes.cattrs&.each do |key, klass_name|
+            next unless hash.key?(key)
+
+            klass = klass_name.constantize
+            klasses_to_reset << klass
+
+            begin
+              attrs = Serializer.deserialize(hash[key])
+              attrs.each do |attr_name, value|
+                klass.public_send(:"#{attr_name}=", value) if klass.respond_to?(:"#{attr_name}=")
+              end
+            rescue => e
+              # Log but don't fail if attributes can't be restored
+              # (e.g., attribute removed between enqueue and execute)
+              Shoryuken.logger.warn("Failed to restore CurrentAttributes #{klass_name}: #{e.message}")
+            end
+          end
+
+          super
+        ensure
+          klasses_to_reset.each(&:reset)
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/active_job/job_wrapper.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'active_job'
+require 'shoryuken/worker'
+
+module Shoryuken
+  module ActiveJob
+    # Internal worker class that processes ActiveJob jobs.
+    # This class bridges ActiveJob's interface with Shoryuken's worker interface.
+    #
+    # @api private
+    class JobWrapper # :nodoc:
+      include Shoryuken::Worker
+
+      shoryuken_options body_parser: :json, auto_delete: true
+
+      # Processes an ActiveJob job from an SQS message.
+      #
+      # @param sqs_msg [Shoryuken::Message] The SQS message containing the job data
+      # @param hash [Hash] The parsed job data from the message body
+      def perform(sqs_msg, hash)
+        receive_count = sqs_msg.attributes['ApproximateReceiveCount'].to_i
+        past_receives = receive_count - 1
+        ::ActiveJob::Base.execute hash.merge({ 'executions' => past_receives })
+      end
+    end
+  end
+end

data/lib/shoryuken/body_parser.rb

@@ -1,8 +1,16 @@
 # frozen_string_literal: true
 
 module Shoryuken
+  # Parses SQS message bodies according to worker configuration.
+  # Supports JSON parsing, text extraction, custom Procs, and
+  # any object that responds to parse or load methods.
   class BodyParser
     class << self
+      # Parses the body of an SQS message according to the worker's body_parser option
+      #
+      # @param worker_class [Class] the worker class with shoryuken options
+      # @param sqs_msg [Shoryuken::Message] the SQS message to parse
+      # @return [Object] the parsed message body
       def parse(worker_class, sqs_msg)
         body_parser = worker_class.get_shoryuken_options['body_parser']
 

data/lib/shoryuken/client.rb

@@ -1,18 +1,32 @@
 # frozen_string_literal: true
 
 module Shoryuken
+  # Client class for interacting with SQS queues.
+  # Provides a simple interface for accessing and managing queue instances.
   class Client
+    # @return [Hash{String => Shoryuken::Queue}] cached queue instances by name
     @@queues = {}
 
     class << self
+      # Returns a Queue instance for the given queue name
+      #
+      # @param name [String, Symbol] the name of the queue
+      # @return [Shoryuken::Queue] the queue instance
       def queues(name)
         @@queues[name.to_s] ||= Shoryuken::Queue.new(sqs, name)
       end
 
+      # Returns the current SQS client
+      #
+      # @return [Aws::SQS::Client] the SQS client
       def sqs
         Shoryuken.sqs_client
       end
 
+      # Sets a new SQS client and clears the queue cache
+      #
+      # @param sqs [Aws::SQS::Client] the new SQS client
+      # @return [Aws::SQS::Client] the SQS client
       def sqs=(sqs)
         # Since the @@queues values (Shoryuken::Queue objects) are built referencing @@sqs, if it changes, we need to
         #   re-build them on subsequent calls to `.queues(name)`.

data/lib/shoryuken/default_exception_handler.rb

@@ -1,9 +1,18 @@
 # frozen_string_literal: true
 
 module Shoryuken
+  # Default exception handler that logs errors during message processing.
+  # Implements a simple error logging strategy that outputs the exception
+  # message and backtrace to the configured logger.
   class DefaultExceptionHandler
     extend Util
 
+    # Handles an exception that occurred during message processing
+    #
+    # @param exception [Exception] the exception that was raised
+    # @param _queue [String] the queue name where the error occurred (unused)
+    # @param _sqs_msg [Shoryuken::Message] the message being processed (unused)
+    # @return [void]
     def self.call(exception, _queue, _sqs_msg)
       logger.error { "Processor failed: #{exception.message}" }
       logger.error { exception.backtrace.join("\n") } if exception.backtrace

data/lib/shoryuken/default_worker_registry.rb

@@ -1,19 +1,34 @@
 # frozen_string_literal: true
 
 module Shoryuken
+  # Default implementation of the worker registry.
+  # Stores and retrieves worker classes mapped to queue names.
   class DefaultWorkerRegistry < WorkerRegistry
+    # Initializes a new DefaultWorkerRegistry with an empty workers hash
     def initialize
       @workers = Shoryuken::Helpers::AtomicHash.new
     end
 
+    # Checks if a queue is configured for batch message receiving
+    #
+    # @param queue [String] the queue name
+    # @return [Boolean] true if the queue's worker has batch mode enabled
     def batch_receive_messages?(queue)
       !!(@workers[queue] && @workers[queue].get_shoryuken_options['batch'])
     end
 
+    # Clears all registered workers
+    #
+    # @return [void]
     def clear
       @workers.clear
     end
 
+    # Fetches a worker instance for processing a message
+    #
+    # @param queue [String] the queue name
+    # @param message [Shoryuken::Message, Array<Shoryuken::Message>] the message or batch
+    # @return [Object, nil] a new worker instance or nil if not found
     def fetch_worker(queue, message)
       worker_class = !message.is_a?(Array) &&
                      message.message_attributes &&
@@ -29,13 +44,22 @@ module Shoryuken
       worker_class.new if worker_class
     end
 
+    # Returns all registered queue names
+    #
+    # @return [Array<String>] the queue names with registered workers
     def queues
       @workers.keys
     end
 
+    # Registers a worker class for a queue
+    #
+    # @param queue [String] the queue name
+    # @param clazz [Class] the worker class to register
+    # @return [Class] the registered worker class
+    # @raise [Errors::InvalidWorkerRegistrationError] if a batchable worker is already registered for the queue
     def register_worker(queue, clazz)
       if (worker_class = @workers[queue]) && (worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true)
-        fail ArgumentError, "Could not register #{clazz} for #{queue}, "\
+        raise Errors::InvalidWorkerRegistrationError, "Could not register #{clazz} for #{queue}, "\
           "because #{worker_class} is already registered for this queue, "\
           "and Shoryuken doesn't support a batchable worker for a queue with multiple workers"
       end
@@ -43,6 +67,10 @@ module Shoryuken
       @workers[queue] = clazz
     end
 
+    # Returns all worker classes for a queue
+    #
+    # @param queue [String] the queue name
+    # @return [Array<Class>] the registered worker classes
     def workers(queue)
       [@workers.fetch(queue, [])].flatten
     end