shoryuken 6.2.1 → 7.0.2

data/lib/shoryuken/helpers/atomic_hash.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # A thread-safe hash implementation using Ruby's Mutex for all operations.
+    #
+    # This class provides a hash-like interface with thread-safe operations, serving as a
+    # drop-in replacement for Concurrent::Hash without requiring external dependencies.
+    # The implementation uses a single mutex to protect both read and write operations,
+    # ensuring complete thread safety across all Ruby implementations including JRuby.
+    #
+    # Since hash operations (lookup, assignment) are very fast, the mutex overhead is
+    # minimal while providing guaranteed safety and simplicity. This approach avoids
+    # the complexity of copy-on-write while maintaining excellent performance for
+    # typical usage patterns.
+    #
+    # @note This implementation uses mutex synchronization for all operations,
+    #   ensuring complete thread safety with minimal performance impact.
+    #
+    # @note All operations are atomic and will never see partial effects from
+    #   concurrent operations.
+    #
+    # @example Basic hash operations
+    #   hash = Shoryuken::Helpers::AtomicHash.new
+    #   hash['key'] = 'value'
+    #   hash['key']           # => 'value'
+    #   hash.keys             # => ['key']
+    #   hash.clear
+    #   hash['key']           # => nil
+    #
+    # @example Worker registry usage
+    #   @workers = Shoryuken::Helpers::AtomicHash.new
+    #
+    #   # Registration (infrequent writes)
+    #   @workers['queue_name'] = WorkerClass
+    #
+    #   # Lookups (frequent reads)
+    #   worker_class = @workers['queue_name']
+    #   available_queues = @workers.keys
+    #   worker_class = @workers.fetch('queue_name', DefaultWorker)
+    #
+    # @example Thread-safe concurrent access
+    #   hash = Shoryuken::Helpers::AtomicHash.new
+    #
+    #   # Multiple threads can safely write
+    #   Thread.new { hash['key1'] = 'value1' }
+    #   Thread.new { hash['key2'] = 'value2' }
+    #
+    #   # Multiple threads can safely read concurrently
+    #   Thread.new { puts hash['key1'] }
+    #   Thread.new { puts hash.keys.size }
+    class AtomicHash
+      # Creates a new empty atomic hash.
+      #
+      # The hash starts empty and ready to accept key-value pairs through
+      # thread-safe operations.
+      #
+      # @return [AtomicHash] A new empty atomic hash instance
+      #
+      # @example Creating an empty hash
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash.keys  # => []
+      def initialize
+        @mutex = Mutex.new
+        @hash = {}
+      end
+
+      # Returns the value associated with the given key.
+      #
+      # This operation is thread-safe and will return a consistent value
+      # even when called concurrently with write operations.
+      #
+      # @param key [Object] The key to look up
+      # @return [Object, nil] The value associated with the key, or nil if not found
+      #
+      # @example Reading values
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['existing'] = 'value'
+      #   hash['existing']    # => 'value'
+      #   hash['missing']     # => nil
+      #
+      # @example Works with any key type
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash[:symbol] = 'symbol_value'
+      #   hash[42] = 'number_value'
+      #   hash[:symbol]  # => 'symbol_value'
+      #   hash[42]       # => 'number_value'
+      def [](key)
+        @mutex.synchronize { @hash[key] }
+      end
+
+      # Sets the value for the given key.
+      #
+      # This is a thread-safe write operation that ensures data integrity
+      # when called concurrently with other read or write operations.
+      #
+      # @param key [Object] The key to set
+      # @param value [Object] The value to associate with the key
+      # @return [Object] The assigned value
+      #
+      # @example Setting values
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['queue1'] = 'Worker1'
+      #   hash['queue2'] = 'Worker2'
+      #   hash['queue1']  # => 'Worker1'
+      #
+      # @example Overwriting values
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['key'] = 'old_value'
+      #   hash['key'] = 'new_value'
+      #   hash['key']  # => 'new_value'
+      def []=(key, value)
+        @mutex.synchronize { @hash[key] = value }
+      end
+
+      # Removes all key-value pairs from the hash.
+      #
+      # This is a thread-safe write operation that ensures atomicity
+      # when called concurrently with other operations.
+      #
+      # @return [Hash] An empty hash (for compatibility with standard Hash#clear)
+      #
+      # @example Clearing all entries
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['key1'] = 'value1'
+      #   hash['key2'] = 'value2'
+      #   hash.keys.size  # => 2
+      #   hash.clear
+      #   hash.keys.size  # => 0
+      #   hash['key1']    # => nil
+      def clear
+        @mutex.synchronize { @hash.clear }
+      end
+
+      # Returns an array of all keys in the hash.
+      #
+      # This operation is thread-safe and will return a consistent snapshot
+      # of keys even when called concurrently with write operations.
+      #
+      # @return [Array] An array containing all keys in the hash
+      #
+      # @example Getting all keys
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['queue1'] = 'Worker1'
+      #   hash['queue2'] = 'Worker2'
+      #   hash.keys  # => ['queue1', 'queue2'] (order not guaranteed)
+      #
+      # @example Empty hash returns empty array
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash.keys  # => []
+      def keys
+        @mutex.synchronize { @hash.keys }
+      end
+
+      # Returns the value for the given key, or a default value if the key is not found.
+      #
+      # This operation is thread-safe and will return a consistent value
+      # even when called concurrently with write operations.
+      #
+      # @param key [Object] The key to look up
+      # @param default [Object] The value to return if the key is not found
+      # @return [Object] The value associated with the key, or the default value
+      #
+      # @example Fetching with defaults
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['existing'] = 'found'
+      #   hash.fetch('existing', 'default')  # => 'found'
+      #   hash.fetch('missing', 'default')   # => 'default'
+      #
+      # @example Default parameter is optional
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash.fetch('missing')  # => nil
+      #
+      # @example Useful for providing fallback collections
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   workers = hash.fetch('queue_name', [])  # => [] if not found
+      def fetch(key, default = nil)
+        @mutex.synchronize { @hash.fetch(key, default) }
+      end
+    end
+  end
+end

data/lib/shoryuken/helpers/hash_utils.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # Utility methods for hash manipulation.
+    #
+    # This module provides helper methods for common hash operations that were
+    # previously implemented as core class extensions. By using a dedicated
+    # helper module, we avoid polluting the global namespace while maintaining
+    # the same functionality.
+    #
+    # @example Basic usage
+    #   hash = { 'key1' => 'value1', 'key2' => { 'nested' => 'value2' } }
+    #   symbolized = Shoryuken::Helpers::HashUtils.deep_symbolize_keys(hash)
+    #   # => { key1: 'value1', key2: { nested: 'value2' } }
+    module HashUtils
+      class << self
+        # Recursively converts hash keys to symbols.
+        #
+        # This method traverses a hash structure and converts all string keys
+        # to symbols, including nested hashes. Non-hash values are left unchanged.
+        # This is useful for normalizing configuration data loaded from YAML files.
+        #
+        # @param hash [Hash, Object] The hash to convert, or any other object
+        # @return [Hash, Object] Hash with symbolized keys, or the original object if not a hash
+        #
+        # @example Converting a simple hash
+        #   hash = { 'key1' => 'value1', 'key2' => 'value2' }
+        #   HashUtils.deep_symbolize_keys(hash)
+        #   # => { key1: 'value1', key2: 'value2' }
+        #
+        # @example Converting a nested hash
+        #   hash = { 'config' => { 'timeout' => 30, 'retries' => 3 } }
+        #   HashUtils.deep_symbolize_keys(hash)
+        #   # => { config: { timeout: 30, retries: 3 } }
+        #
+        # @example Handling non-hash input gracefully
+        #   HashUtils.deep_symbolize_keys('not a hash')
+        #   # => 'not a hash'
+        #
+        # @example Mixed value types
+        #   hash = { 'string' => 'value', 'number' => 42, 'nested' => { 'bool' => true } }
+        #   HashUtils.deep_symbolize_keys(hash)
+        #   # => { string: 'value', number: 42, nested: { bool: true } }
+        def deep_symbolize_keys(hash)
+          return hash unless hash.is_a?(Hash)
+
+          hash.each_with_object({}) do |(key, value), result|
+            symbol_key = key.is_a?(String) ? key.to_sym : key
+            result[symbol_key] = value.is_a?(Hash) ? deep_symbolize_keys(value) : value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/helpers/string_utils.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # Utility methods for string manipulation.
+    #
+    # This module provides helper methods for common string operations that were
+    # previously implemented as core class extensions. By using a dedicated
+    # helper module, we avoid polluting the global namespace while maintaining
+    # the same functionality.
+    #
+    # @example Basic usage
+    #   klass = Shoryuken::Helpers::StringUtils.constantize('MyWorker')
+    #   # => MyWorker
+    module StringUtils
+      class << self
+        # Converts a string to a constant.
+        #
+        # This method takes a string representation of a constant name and returns
+        # the actual constant. It handles nested constants (e.g., 'Foo::Bar') and
+        # leading double colons (e.g., '::Object'). This is commonly used for
+        # dynamically loading worker classes from configuration.
+        #
+        # @param string [String] The string to convert to a constant
+        # @return [Class, Module] The constant represented by the string
+        # @raise [NameError] if the constant is not found or not defined
+        #
+        # @example Converting a simple class name
+        #   StringUtils.constantize('String')
+        #   # => String
+        #
+        # @example Converting a nested constant
+        #   StringUtils.constantize('Shoryuken::Worker')
+        #   # => Shoryuken::Worker
+        #
+        # @example Handling leading double colon
+        #   StringUtils.constantize('::Object')
+        #   # => Object
+        #
+        # @example Worker class loading
+        #   worker_class = StringUtils.constantize('MyApp::EmailWorker')
+        #   worker_instance = worker_class.new
+        #
+        # @example Error handling
+        #   begin
+        #     StringUtils.constantize('NonExistentClass')
+        #   rescue NameError => e
+        #     puts "Class not found: #{e.message}"
+        #   end
+        def constantize(string)
+          names = string.split('::')
+          names.shift if names.empty? || names.first.empty?
+
+          constant = Object
+
+          names.each do |name|
+            constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+          end
+
+          constant
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/helpers/timer_task.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # A thread-safe timer task implementation.
+    # Drop-in replacement for Concurrent::TimerTask without external dependencies.
+    class TimerTask
+      # Initializes a new TimerTask
+      #
+      # @param execution_interval [Float] interval in seconds between task executions
+      # @param task [Proc] the task to execute on each interval (provided as a block)
+      # @return [TimerTask] a new TimerTask instance
+      # @raise [ArgumentError] if no block is provided or interval is not positive
+      # @yield the task to execute on each interval
+      def initialize(execution_interval:, &task)
+        raise ArgumentError, 'A block must be provided' unless block_given?
+
+        @execution_interval = Float(execution_interval)
+        raise ArgumentError, 'execution_interval must be positive' if @execution_interval <= 0
+
+        @task = task
+        @mutex = Mutex.new
+        @thread = nil
+        @running = false
+        @killed = false
+      end
+
+      # Starts the timer task execution
+      #
+      # @return [TimerTask] self for method chaining
+      def execute
+        @mutex.synchronize do
+          return self if @running || @killed
+
+          @running = true
+          @thread = Thread.new { run_timer_loop }
+        end
+        self
+      end
+
+      # Stops and kills the timer task
+      #
+      # @return [Boolean] true if killed, false if already killed
+      def kill
+        @mutex.synchronize do
+          return false if @killed
+
+          @killed = true
+          @running = false
+
+          @thread.kill if @thread&.alive?
+        end
+        true
+      end
+
+      private
+
+      # Runs the timer loop in a separate thread
+      #
+      # @return [void]
+      def run_timer_loop
+        until @killed
+          sleep(@execution_interval)
+          break if @killed
+
+          begin
+            @task.call
+          rescue => e
+            # Log the error but continue running
+            # This matches the behavior of Concurrent::TimerTask
+            warn "TimerTask execution error: #{e.message}"
+            warn e.backtrace.join("\n") if e.backtrace
+          end
+        end
+      ensure
+        @mutex.synchronize { @running = false }
+      end
+    end
+  end
+end

data/lib/shoryuken/inline_message.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  # A high-performance alternative to OpenStruct for representing SQS messages.
+  #
+  # InlineMessage is a Struct-based implementation that provides the same interface
+  # as the previous OpenStruct-based message representation but with significantly
+  # better performance characteristics. It contains all the essential attributes
+  # needed to represent an Amazon SQS message within the Shoryuken framework.
+  InlineMessage = Struct.new(
+    :body,
+    :attributes,
+    :md5_of_body,
+    :md5_of_message_attributes,
+    :message_attributes,
+    :message_id,
+    :receipt_handle,
+    :delete,
+    :queue_name,
+    keyword_init: true
+  )
+end

data/lib/shoryuken/launcher.rb

@@ -1,11 +1,31 @@
+# frozen_string_literal: true
+
 module Shoryuken
+  # Launches and coordinates Shoryuken's message processing managers.
+  # Handles the lifecycle of the processing system including startup, shutdown, and health checks.
   class Launcher
     include Util
 
+    # Initializes a new Launcher with managers for each processing group
     def initialize
       @managers = create_managers
+      @stopping = false
+    end
+
+    # Indicates whether the launcher is in the process of stopping.
+    #
+    # This flag is set to true when either {#stop} or {#stop!} is called,
+    # and is used by ActiveJob adapters to signal jobs that they should
+    # checkpoint and prepare for graceful shutdown.
+    #
+    # @return [Boolean] true if stopping, false otherwise
+    def stopping?
+      @stopping
     end
 
+    # Starts the message processing system
+    #
+    # @return [void]
     def start
       logger.info { 'Starting' }
 
@@ -13,7 +33,11 @@ module Shoryuken
       start_managers
     end
 
+    # Forces an immediate stop of all processing
+    #
+    # @return [void]
     def stop!
+      @stopping = true
       initiate_stop
 
       # Don't await here so the timeout below is not delayed
@@ -25,7 +49,11 @@ module Shoryuken
       fire_event(:stopped)
     end
 
+    # Gracefully stops all processing, waiting for in-flight messages
+    #
+    # @return [void]
     def stop
+      @stopping = true
       fire_event(:quiet, true)
 
       initiate_stop
@@ -39,6 +67,9 @@ module Shoryuken
       fire_event(:stopped)
     end
 
+    # Checks if all processing groups are healthy
+    #
+    # @return [Boolean] true if all groups are running normally
     def healthy?
       Shoryuken.groups.keys.all? do |group|
         manager = @managers.find { |m| m.group == group }
@@ -48,30 +79,48 @@ module Shoryuken
 
     private
 
+    # Stops all managers from dispatching new messages
+    #
+    # @return [void]
     def stop_new_dispatching
       @managers.each(&:stop_new_dispatching)
     end
 
+    # Waits for any in-progress dispatching to complete
+    #
+    # @return [void]
     def await_dispatching_in_progress
       @managers.each(&:await_dispatching_in_progress)
     end
 
+    # Returns the executor for running async operations
+    #
+    # @return [Concurrent::ExecutorService] the executor service
     def executor
       @_executor ||= Shoryuken.launcher_executor || Concurrent.global_io_executor
     end
 
+    # Starts all managers in parallel futures
+    #
+    # @return [void]
     def start_managers
       @managers.each do |manager|
         Concurrent::Future.execute { manager.start }
       end
     end
 
+    # Initiates the stop sequence
+    #
+    # @return [void]
     def initiate_stop
       logger.info { 'Shutting down' }
 
       stop_callback
     end
 
+    # Executes the start callback and fires startup event
+    #
+    # @return [void]
     def start_callback
       if (callback = Shoryuken.start_callback)
         logger.debug { 'Calling start_callback' }
@@ -81,6 +130,9 @@ module Shoryuken
       fire_event(:startup)
     end
 
+    # Executes the stop callback and fires shutdown event
+    #
+    # @return [void]
     def stop_callback
       if (callback = Shoryuken.stop_callback)
         logger.debug { 'Calling stop_callback' }
@@ -90,6 +142,9 @@ module Shoryuken
       fire_event(:shutdown, true)
     end
 
+    # Creates managers for each configured processing group
+    #
+    # @return [Array<Shoryuken::Manager>] the created managers
     def create_managers
       Shoryuken.groups.map do |group, options|
         Shoryuken::Manager.new(

data/lib/shoryuken/logging/base.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Logging
+    # Base formatter class that provides common functionality for Shoryuken log formatters.
+    # Provides thread ID generation and context management.
+    class Base < ::Logger::Formatter
+      # Generates a thread ID for the current thread.
+      # Uses a combination of thread object_id and process ID to create a unique identifier.
+      #
+      # @return [String] A base36-encoded thread identifier
+      def tid
+        Thread.current['shoryuken_tid'] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
+      end
+
+      # Returns the current logging context as a formatted string.
+      # Context is set using {Shoryuken::Logging.with_context}.
+      #
+      # @return [String] Formatted context string or empty string if no context
+      def context
+        c = Shoryuken::Logging.current_context
+        c ? " #{c}" : ''
+      end
+    end
+  end
+end

data/lib/shoryuken/logging/pretty.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Logging
+    # A pretty log formatter that includes timestamps, process ID, thread ID,
+    # context information, and severity in a human-readable format.
+    #
+    # Output format: "TIMESTAMP PID TID-THREAD_ID CONTEXT SEVERITY: MESSAGE"
+    #
+    # @example Sample output
+    #   # 2023-08-15T10:30:45Z 12345 TID-abc123 MyWorker/queue1/msg-456 INFO: Processing message
+    class Pretty < Base
+      # Formats a log message with timestamp and full context information.
+      #
+      # @param severity [String] Log severity level (DEBUG, INFO, WARN, ERROR, FATAL)
+      # @param time [Time] Timestamp when the log entry was created
+      # @param _program_name [String] Program name (unused)
+      # @param message [String] The log message
+      # @return [String] Formatted log entry with newline
+      def call(severity, time, _program_name, message)
+        "#{time.utc.iso8601} #{Process.pid} TID-#{tid}#{context} #{severity}: #{message}\n"
+      end
+    end
+  end
+end

data/lib/shoryuken/logging/without_timestamp.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Logging
+    # A log formatter that excludes timestamps from output.
+    # Useful for environments where timestamps are added by external logging systems.
+    #
+    # Output format: "pid=PID tid=THREAD_ID CONTEXT SEVERITY: MESSAGE"
+    #
+    # @example Sample output
+    #   # pid=12345 tid=abc123 MyWorker/queue1/msg-456 INFO: Processing message
+    class WithoutTimestamp < Base
+      # Formats a log message without timestamp information.
+      #
+      # @param severity [String] Log severity level (DEBUG, INFO, WARN, ERROR, FATAL)
+      # @param _time [Time] Timestamp (unused)
+      # @param _program_name [String] Program name (unused)
+      # @param message [String] The log message
+      # @return [String] Formatted log entry with newline
+      def call(severity, _time, _program_name, message)
+        "pid=#{Process.pid} tid=#{tid}#{context} #{severity}: #{message}\n"
+      end
+    end
+  end
+end