shoryuken 7.0.0.alpha1 → 7.0.0.alpha2

data/lib/shoryuken/helpers/atomic_hash.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # A thread-safe hash implementation using Ruby's Mutex for all operations.
+    #
+    # This class provides a hash-like interface with thread-safe operations, serving as a
+    # drop-in replacement for Concurrent::Hash without requiring external dependencies.
+    # The implementation uses a single mutex to protect both read and write operations,
+    # ensuring complete thread safety across all Ruby implementations including JRuby.
+    #
+    # Since hash operations (lookup, assignment) are very fast, the mutex overhead is
+    # minimal while providing guaranteed safety and simplicity. This approach avoids
+    # the complexity of copy-on-write while maintaining excellent performance for
+    # typical usage patterns.
+    #
+    # @note This implementation uses mutex synchronization for all operations,
+    #   ensuring complete thread safety with minimal performance impact.
+    #
+    # @note All operations are atomic and will never see partial effects from
+    #   concurrent operations.
+    #
+    # @example Basic hash operations
+    #   hash = Shoryuken::Helpers::AtomicHash.new
+    #   hash['key'] = 'value'
+    #   hash['key']           # => 'value'
+    #   hash.keys             # => ['key']
+    #   hash.clear
+    #   hash['key']           # => nil
+    #
+    # @example Worker registry usage
+    #   @workers = Shoryuken::Helpers::AtomicHash.new
+    #
+    #   # Registration (infrequent writes)
+    #   @workers['queue_name'] = WorkerClass
+    #
+    #   # Lookups (frequent reads)
+    #   worker_class = @workers['queue_name']
+    #   available_queues = @workers.keys
+    #   worker_class = @workers.fetch('queue_name', DefaultWorker)
+    #
+    # @example Thread-safe concurrent access
+    #   hash = Shoryuken::Helpers::AtomicHash.new
+    #
+    #   # Multiple threads can safely write
+    #   Thread.new { hash['key1'] = 'value1' }
+    #   Thread.new { hash['key2'] = 'value2' }
+    #
+    #   # Multiple threads can safely read concurrently
+    #   Thread.new { puts hash['key1'] }
+    #   Thread.new { puts hash.keys.size }
+    class AtomicHash
+      # Creates a new empty atomic hash.
+      #
+      # The hash starts empty and ready to accept key-value pairs through
+      # thread-safe operations.
+      #
+      # @return [AtomicHash] A new empty atomic hash instance
+      #
+      # @example Creating an empty hash
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash.keys  # => []
+      def initialize
+        @mutex = Mutex.new
+        @hash = {}
+      end
+
+      # Returns the value associated with the given key.
+      #
+      # This operation is thread-safe and will return a consistent value
+      # even when called concurrently with write operations.
+      #
+      # @param key [Object] The key to look up
+      # @return [Object, nil] The value associated with the key, or nil if not found
+      #
+      # @example Reading values
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['existing'] = 'value'
+      #   hash['existing']    # => 'value'
+      #   hash['missing']     # => nil
+      #
+      # @example Works with any key type
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash[:symbol] = 'symbol_value'
+      #   hash[42] = 'number_value'
+      #   hash[:symbol]  # => 'symbol_value'
+      #   hash[42]       # => 'number_value'
+      def [](key)
+        @mutex.synchronize { @hash[key] }
+      end
+
+      # Sets the value for the given key.
+      #
+      # This is a thread-safe write operation that ensures data integrity
+      # when called concurrently with other read or write operations.
+      #
+      # @param key [Object] The key to set
+      # @param value [Object] The value to associate with the key
+      # @return [Object] The assigned value
+      #
+      # @example Setting values
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['queue1'] = 'Worker1'
+      #   hash['queue2'] = 'Worker2'
+      #   hash['queue1']  # => 'Worker1'
+      #
+      # @example Overwriting values
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['key'] = 'old_value'
+      #   hash['key'] = 'new_value'
+      #   hash['key']  # => 'new_value'
+      def []=(key, value)
+        @mutex.synchronize { @hash[key] = value }
+      end
+
+      # Removes all key-value pairs from the hash.
+      #
+      # This is a thread-safe write operation that ensures atomicity
+      # when called concurrently with other operations.
+      #
+      # @return [Hash] An empty hash (for compatibility with standard Hash#clear)
+      #
+      # @example Clearing all entries
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['key1'] = 'value1'
+      #   hash['key2'] = 'value2'
+      #   hash.keys.size  # => 2
+      #   hash.clear
+      #   hash.keys.size  # => 0
+      #   hash['key1']    # => nil
+      def clear
+        @mutex.synchronize { @hash.clear }
+      end
+
+      # Returns an array of all keys in the hash.
+      #
+      # This operation is thread-safe and will return a consistent snapshot
+      # of keys even when called concurrently with write operations.
+      #
+      # @return [Array] An array containing all keys in the hash
+      #
+      # @example Getting all keys
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['queue1'] = 'Worker1'
+      #   hash['queue2'] = 'Worker2'
+      #   hash.keys  # => ['queue1', 'queue2'] (order not guaranteed)
+      #
+      # @example Empty hash returns empty array
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash.keys  # => []
+      def keys
+        @mutex.synchronize { @hash.keys }
+      end
+
+      # Returns the value for the given key, or a default value if the key is not found.
+      #
+      # This operation is thread-safe and will return a consistent value
+      # even when called concurrently with write operations.
+      #
+      # @param key [Object] The key to look up
+      # @param default [Object] The value to return if the key is not found
+      # @return [Object] The value associated with the key, or the default value
+      #
+      # @example Fetching with defaults
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['existing'] = 'found'
+      #   hash.fetch('existing', 'default')  # => 'found'
+      #   hash.fetch('missing', 'default')   # => 'default'
+      #
+      # @example Default parameter is optional
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash.fetch('missing')  # => nil
+      #
+      # @example Useful for providing fallback collections
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   workers = hash.fetch('queue_name', [])  # => [] if not found
+      def fetch(key, default = nil)
+        @mutex.synchronize { @hash.fetch(key, default) }
+      end
+    end
+  end
+end

data/lib/shoryuken/helpers/hash_utils.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # Utility methods for hash manipulation.
+    #
+    # This module provides helper methods for common hash operations that were
+    # previously implemented as core class extensions. By using a dedicated
+    # helper module, we avoid polluting the global namespace while maintaining
+    # the same functionality.
+    #
+    # @example Basic usage
+    #   hash = { 'key1' => 'value1', 'key2' => { 'nested' => 'value2' } }
+    #   symbolized = Shoryuken::Helpers::HashUtils.deep_symbolize_keys(hash)
+    #   # => { key1: 'value1', key2: { nested: 'value2' } }
+    module HashUtils
+      class << self
+        # Recursively converts hash keys to symbols.
+        #
+        # This method traverses a hash structure and converts all string keys
+        # to symbols, including nested hashes. Non-hash values are left unchanged.
+        # This is useful for normalizing configuration data loaded from YAML files.
+        #
+        # @param hash [Hash, Object] The hash to convert, or any other object
+        # @return [Hash, Object] Hash with symbolized keys, or the original object if not a hash
+        #
+        # @example Converting a simple hash
+        #   hash = { 'key1' => 'value1', 'key2' => 'value2' }
+        #   HashUtils.deep_symbolize_keys(hash)
+        #   # => { key1: 'value1', key2: 'value2' }
+        #
+        # @example Converting a nested hash
+        #   hash = { 'config' => { 'timeout' => 30, 'retries' => 3 } }
+        #   HashUtils.deep_symbolize_keys(hash)
+        #   # => { config: { timeout: 30, retries: 3 } }
+        #
+        # @example Handling non-hash input gracefully
+        #   HashUtils.deep_symbolize_keys('not a hash')
+        #   # => 'not a hash'
+        #
+        # @example Mixed value types
+        #   hash = { 'string' => 'value', 'number' => 42, 'nested' => { 'bool' => true } }
+        #   HashUtils.deep_symbolize_keys(hash)
+        #   # => { string: 'value', number: 42, nested: { bool: true } }
+        def deep_symbolize_keys(hash)
+          return hash unless hash.is_a?(Hash)
+
+          hash.each_with_object({}) do |(key, value), result|
+            symbol_key = key.is_a?(String) ? key.to_sym : key
+            result[symbol_key] = value.is_a?(Hash) ? deep_symbolize_keys(value) : value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/helpers/string_utils.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # Utility methods for string manipulation.
+    #
+    # This module provides helper methods for common string operations that were
+    # previously implemented as core class extensions. By using a dedicated
+    # helper module, we avoid polluting the global namespace while maintaining
+    # the same functionality.
+    #
+    # @example Basic usage
+    #   klass = Shoryuken::Helpers::StringUtils.constantize('MyWorker')
+    #   # => MyWorker
+    module StringUtils
+      class << self
+        # Converts a string to a constant.
+        #
+        # This method takes a string representation of a constant name and returns
+        # the actual constant. It handles nested constants (e.g., 'Foo::Bar') and
+        # leading double colons (e.g., '::Object'). This is commonly used for
+        # dynamically loading worker classes from configuration.
+        #
+        # @param string [String] The string to convert to a constant
+        # @return [Class, Module] The constant represented by the string
+        # @raise [NameError] if the constant is not found or not defined
+        #
+        # @example Converting a simple class name
+        #   StringUtils.constantize('String')
+        #   # => String
+        #
+        # @example Converting a nested constant
+        #   StringUtils.constantize('Shoryuken::Worker')
+        #   # => Shoryuken::Worker
+        #
+        # @example Handling leading double colon
+        #   StringUtils.constantize('::Object')
+        #   # => Object
+        #
+        # @example Worker class loading
+        #   worker_class = StringUtils.constantize('MyApp::EmailWorker')
+        #   worker_instance = worker_class.new
+        #
+        # @example Error handling
+        #   begin
+        #     StringUtils.constantize('NonExistentClass')
+        #   rescue NameError => e
+        #     puts "Class not found: #{e.message}"
+        #   end
+        def constantize(string)
+          names = string.split('::')
+          names.shift if names.empty? || names.first.empty?
+
+          constant = Object
+
+          names.each do |name|
+            constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+          end
+
+          constant
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/inline_message.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  # A high-performance alternative to OpenStruct for representing SQS messages.
+  #
+  # InlineMessage is a Struct-based implementation that provides the same interface
+  # as the previous OpenStruct-based message representation but with significantly
+  # better performance characteristics. It contains all the essential attributes
+  # needed to represent an Amazon SQS message within the Shoryuken framework.
+  InlineMessage = Struct.new(
+    :body,
+    :attributes,
+    :md5_of_body,
+    :md5_of_message_attributes,
+    :message_attributes,
+    :message_id,
+    :receipt_handle,
+    :delete,
+    :queue_name,
+    keyword_init: true
+  )
+end

data/lib/shoryuken/launcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class Launcher
     include Util

data/lib/shoryuken/logging.rb

@@ -1,12 +1,13 @@
+# frozen_string_literal: true
+
 require 'time'
 require 'logger'
 
 module Shoryuken
   module Logging
-    class Pretty < Logger::Formatter
-      # Provide a call() method that returns the formatted message.
-      def call(severity, time, _program_name, message)
-        "#{time.utc.iso8601} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
+    class Base < ::Logger::Formatter
+      def tid
+        Thread.current['shoryuken_tid'] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
       end
 
       def context
@@ -15,6 +16,19 @@ module Shoryuken
       end
     end
 
+    class Pretty < Base
+      # Provide a call() method that returns the formatted message.
+      def call(severity, time, _program_name, message)
+        "#{time.utc.iso8601} #{Process.pid} TID-#{tid}#{context} #{severity}: #{message}\n"
+      end
+    end
+
+    class WithoutTimestamp < Base
+      def call(severity, _time, _program_name, message)
+        "pid=#{Process.pid} tid=#{tid}#{context} #{severity}: #{message}\n"
+      end
+    end
+
     def self.with_context(msg)
       Thread.current[:shoryuken_context] = msg
       yield
@@ -34,7 +48,7 @@ module Shoryuken
     end
 
     def self.logger=(log)
-      @logger = (log || Logger.new('/dev/null'))
+      @logger = log || Logger.new('/dev/null')
     end
   end
 end

data/lib/shoryuken/manager.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class Manager
     include Util
 
     BATCH_LIMIT = 10
-    # See https://github.com/phstc/shoryuken/issues/348#issuecomment-292847028
+    # See https://github.com/ruby-shoryuken/shoryuken/issues/348#issuecomment-292847028
     MIN_DISPATCH_INTERVAL = 0.1
 
     attr_reader :group
@@ -13,10 +15,10 @@ module Shoryuken
       @fetcher                    = fetcher
       @polling_strategy           = polling_strategy
       @max_processors             = concurrency
-      @busy_processors            = Concurrent::AtomicFixnum.new(0)
+      @busy_processors            = Shoryuken::Helpers::AtomicCounter.new(0)
       @executor                   = executor
-      @running                    = Concurrent::AtomicBoolean.new(true)
-      @stop_new_dispatching       = Concurrent::AtomicBoolean.new(false)
+      @running                    = Shoryuken::Helpers::AtomicBoolean.new(true)
+      @stop_new_dispatching       = Shoryuken::Helpers::AtomicBoolean.new(false)
       @dispatching_release_signal = ::Queue.new
     end
 

data/lib/shoryuken/message.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class Message
     extend Forwardable

data/lib/shoryuken/middleware/chain.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   # Middleware is code configured to run before/after
   # a message is processed.  It is patterned after Rack

data/lib/shoryuken/middleware/server/active_record.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server

data/lib/shoryuken/middleware/server/auto_delete.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server

data/lib/shoryuken/middleware/server/auto_extend_visibility.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server
@@ -29,16 +31,14 @@ module Shoryuken
             queue_visibility_timeout = Shoryuken::Client.queues(queue).visibility_timeout
 
             Concurrent::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
-              begin
-                logger.debug do
-                  "Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
-                end
-
-                sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
-              rescue => ex
-                logger.error do
-                  "Could not auto extend the message #{queue}/#{sqs_msg.message_id} visibility timeout. Error: #{ex.message}"
-                end
+              logger.debug do
+                "Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
+              end
+
+              sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
+            rescue => e
+              logger.error do
+                "Could not auto extend the message #{queue}/#{sqs_msg.message_id} visibility timeout. Error: #{e.message}"
               end
             end
           end

data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server
@@ -14,7 +16,7 @@ module Shoryuken
 
           started_at = Time.now
           yield
-        rescue => ex
+        rescue => e
           retry_intervals = worker.class.get_shoryuken_options['retry_intervals']
 
           if retry_intervals.nil? || !handle_failure(sqs_msg, started_at, retry_intervals)
@@ -23,9 +25,9 @@ module Shoryuken
             raise
           end
 
-          logger.warn { "Message #{sqs_msg.message_id} will attempt retry due to error: #{ex.message}" }
+          logger.warn { "Message #{sqs_msg.message_id} will attempt retry due to error: #{e.message}" }
           # since we didn't raise, lets log the backtrace for debugging purposes.
-          logger.debug { ex.backtrace.join("\n") } unless ex.backtrace.nil?
+          logger.debug { e.backtrace.join("\n") } unless e.backtrace.nil?
         end
 
         private

data/lib/shoryuken/middleware/server/timing.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server

data/lib/shoryuken/options.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class Options
     DEFAULTS = {
@@ -17,10 +19,12 @@ module Shoryuken
       }
     }.freeze
 
-    attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout, :groups,
-                  :launcher_executor, :reloader, :enable_reloading,
-                  :start_callback, :stop_callback, :worker_executor, :worker_registry, :exception_handlers
-    attr_writer :default_worker_options, :sqs_client
+    attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout,
+                  :groups, :launcher_executor, :reloader, :enable_reloading,
+                  :start_callback, :stop_callback, :worker_executor, :worker_registry,
+                  :exception_handlers
+
+    attr_writer :default_worker_options, :sqs_client, :logger
     attr_reader :sqs_client_receive_message_opts
 
     def initialize
@@ -96,7 +100,7 @@ module Shoryuken
     end
 
     def logger
-      Shoryuken::Logging.logger
+      @logger ||= Shoryuken::Logging.logger
     end
 
     def thread_priority

data/lib/shoryuken/polling/base_strategy.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Polling
+    # Abstract base class for queue polling strategies.
+    #
+    # This class defines the interface that all polling strategies must implement
+    # to manage queue selection and message flow control in Shoryuken workers.
+    # Polling strategies determine which queue to fetch messages from next and
+    # how to handle scenarios where queues have no messages available.
+    #
+    # @abstract Subclass and override {#next_queue}, {#messages_found}, and {#active_queues}
+    #   to implement a custom polling strategy.
+    #
+    # @example Implementing a custom polling strategy
+    #   class CustomStrategy < BaseStrategy
+    #     def initialize(queues)
+    #       @queues = queues
+    #     end
+    #
+    #     def next_queue
+    #       # Return next queue to poll
+    #       @queues.sample
+    #     end
+    #
+    #     def messages_found(queue, count)
+    #       # Handle result of polling
+    #       logger.info "Found #{count} messages in #{queue}"
+    #     end
+    #
+    #     def active_queues
+    #       # Return list of active queues
+    #       @queues
+    #     end
+    #   end
+    #
+    # @see WeightedRoundRobin
+    # @see StrictPriority
+    class BaseStrategy
+      include Util
+
+      # Returns the next queue to poll for messages.
+      #
+      # This method should return a QueueConfiguration object representing
+      # the next queue that should be polled for messages, or nil if no
+      # queues are currently available for polling.
+      #
+      # @abstract Subclasses must implement this method
+      # @return [QueueConfiguration, nil] Next queue to poll, or nil if none available
+      # @raise [NotImplementedError] if not implemented by subclass
+      def next_queue
+        fail NotImplementedError
+      end
+
+      # Called when messages are found (or not found) in a queue.
+      #
+      # This method is invoked after polling a queue to inform the strategy
+      # about the number of messages that were retrieved. Strategies can use
+      # this information to make decisions about future polling behavior,
+      # such as pausing empty queues or adjusting queue weights.
+      #
+      # @abstract Subclasses must implement this method
+      # @param _queue [String] The name of the queue that was polled
+      # @param _messages_found [Integer] The number of messages retrieved
+      # @raise [NotImplementedError] if not implemented by subclass
+      def messages_found(_queue, _messages_found)
+        fail NotImplementedError
+      end
+
+      # Called when a message from a queue has been processed.
+      #
+      # This optional callback is invoked after a message has been successfully
+      # processed by a worker. Strategies can use this information for cleanup
+      # or to adjust their polling behavior.
+      #
+      # @param _queue [String] The name of the queue whose message was processed
+      # @return [void]
+      def message_processed(_queue); end
+
+      # Returns the list of currently active queues.
+      #
+      # This method should return an array representing the queues that are
+      # currently active and available for polling. The format may vary by
+      # strategy implementation.
+      #
+      # @abstract Subclasses must implement this method
+      # @return [Array] List of active queues
+      # @raise [NotImplementedError] if not implemented by subclass
+      def active_queues
+        fail NotImplementedError
+      end
+
+      # Compares this strategy with another object for equality.
+      #
+      # Two strategies are considered equal if they have the same active queues.
+      # This method also supports comparison with Array objects for backward
+      # compatibility.
+      #
+      # @param other [Object] Object to compare with
+      # @return [Boolean] true if strategies are equivalent
+      def ==(other)
+        case other
+        when Array
+          @queues == other
+        else
+          if other.respond_to?(:active_queues)
+            active_queues == other.active_queues
+          else
+            false
+          end
+        end
+      end
+
+      # Returns the delay time for pausing empty queues.
+      #
+      # This method returns the amount of time (in seconds) that empty queues
+      # should be paused before being polled again. The delay can be set at
+      # the strategy level or falls back to the global Shoryuken delay setting.
+      #
+      # @return [Float] Delay time in seconds
+      def delay
+        @delay || Shoryuken.options[:delay].to_f
+      end
+    end
+  end
+end