shoryuken 6.2.1 → 7.0.0.alpha2

data/lib/shoryuken/helpers/atomic_counter.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # A thread-safe counter implementation using Ruby's Mutex.
+    #
+    # This class provides atomic operations for incrementing, decrementing, and reading
+    # integer values in a thread-safe manner. It serves as a drop-in replacement for
+    # Concurrent::AtomicFixnum without requiring external dependencies.
+    #
+    # The implementation uses a Mutex to ensure thread safety across all Ruby
+    # implementations including JRuby, where true parallelism makes atomic operations
+    # critical for data integrity.
+    #
+    # @note This class is optimized for scenarios with frequent atomic updates
+    #   and occasional reads, such as tracking active worker counts.
+    #
+    # @example Basic usage
+    #   counter = Shoryuken::Helpers::AtomicCounter.new(0)
+    #   counter.increment  # => 1
+    #   counter.increment  # => 2
+    #   counter.value      # => 2
+    #   counter.decrement  # => 1
+    #
+    # @example Tracking busy processors
+    #   @busy_processors = Shoryuken::Helpers::AtomicCounter.new(0)
+    #
+    #   # When starting work
+    #   @busy_processors.increment
+    #
+    #   # When work is done
+    #   @busy_processors.decrement
+    #
+    #   # Check current load
+    #   current_busy = @busy_processors.value
+    class AtomicCounter
+      # Creates a new atomic counter with the specified initial value.
+      #
+      # @param initial_value [Integer] The starting value for the counter
+      # @return [AtomicCounter] A new atomic counter instance
+      #
+      # @example Create counter starting at zero
+      #   counter = Shoryuken::Helpers::AtomicCounter.new
+      #   counter.value  # => 0
+      #
+      # @example Create counter with custom initial value
+      #   counter = Shoryuken::Helpers::AtomicCounter.new(100)
+      #   counter.value  # => 100
+      def initialize(initial_value = 0)
+        @mutex = Mutex.new
+        @value = initial_value
+      end
+
+      # Returns the current value of the counter.
+      #
+      # This operation is thread-safe and will return a consistent value
+      # even when called concurrently with increment/decrement operations.
+      #
+      # @return [Integer] The current counter value
+      #
+      # @example Reading the current value
+      #   counter = Shoryuken::Helpers::AtomicCounter.new(42)
+      #   counter.value  # => 42
+      def value
+        @mutex.synchronize { @value }
+      end
+
+      # Atomically increments the counter by 1 and returns the new value.
+      #
+      # This operation is thread-safe and can be called concurrently from
+      # multiple threads without risk of data corruption or lost updates.
+      #
+      # @return [Integer] The new counter value after incrementing
+      #
+      # @example Incrementing the counter
+      #   counter = Shoryuken::Helpers::AtomicCounter.new(5)
+      #   counter.increment  # => 6
+      #   counter.increment  # => 7
+      def increment
+        @mutex.synchronize { @value += 1 }
+      end
+
+      # Atomically decrements the counter by 1 and returns the new value.
+      #
+      # This operation is thread-safe and can be called concurrently from
+      # multiple threads without risk of data corruption or lost updates.
+      # The counter can go negative if decremented below zero.
+      #
+      # @return [Integer] The new counter value after decrementing
+      #
+      # @example Decrementing the counter
+      #   counter = Shoryuken::Helpers::AtomicCounter.new(5)
+      #   counter.decrement  # => 4
+      #   counter.decrement  # => 3
+      #
+      # @example Counter can go negative
+      #   counter = Shoryuken::Helpers::AtomicCounter.new(0)
+      #   counter.decrement  # => -1
+      def decrement
+        @mutex.synchronize { @value -= 1 }
+      end
+    end
+  end
+end

data/lib/shoryuken/helpers/atomic_hash.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # A thread-safe hash implementation using Ruby's Mutex for all operations.
+    #
+    # This class provides a hash-like interface with thread-safe operations, serving as a
+    # drop-in replacement for Concurrent::Hash without requiring external dependencies.
+    # The implementation uses a single mutex to protect both read and write operations,
+    # ensuring complete thread safety across all Ruby implementations including JRuby.
+    #
+    # Since hash operations (lookup, assignment) are very fast, the mutex overhead is
+    # minimal while providing guaranteed safety and simplicity. This approach avoids
+    # the complexity of copy-on-write while maintaining excellent performance for
+    # typical usage patterns.
+    #
+    # @note This implementation uses mutex synchronization for all operations,
+    #   ensuring complete thread safety with minimal performance impact.
+    #
+    # @note All operations are atomic and will never see partial effects from
+    #   concurrent operations.
+    #
+    # @example Basic hash operations
+    #   hash = Shoryuken::Helpers::AtomicHash.new
+    #   hash['key'] = 'value'
+    #   hash['key']           # => 'value'
+    #   hash.keys             # => ['key']
+    #   hash.clear
+    #   hash['key']           # => nil
+    #
+    # @example Worker registry usage
+    #   @workers = Shoryuken::Helpers::AtomicHash.new
+    #
+    #   # Registration (infrequent writes)
+    #   @workers['queue_name'] = WorkerClass
+    #
+    #   # Lookups (frequent reads)
+    #   worker_class = @workers['queue_name']
+    #   available_queues = @workers.keys
+    #   worker_class = @workers.fetch('queue_name', DefaultWorker)
+    #
+    # @example Thread-safe concurrent access
+    #   hash = Shoryuken::Helpers::AtomicHash.new
+    #
+    #   # Multiple threads can safely write
+    #   Thread.new { hash['key1'] = 'value1' }
+    #   Thread.new { hash['key2'] = 'value2' }
+    #
+    #   # Multiple threads can safely read concurrently
+    #   Thread.new { puts hash['key1'] }
+    #   Thread.new { puts hash.keys.size }
+    class AtomicHash
+      # Creates a new empty atomic hash.
+      #
+      # The hash starts empty and ready to accept key-value pairs through
+      # thread-safe operations.
+      #
+      # @return [AtomicHash] A new empty atomic hash instance
+      #
+      # @example Creating an empty hash
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash.keys  # => []
+      def initialize
+        @mutex = Mutex.new
+        @hash = {}
+      end
+
+      # Returns the value associated with the given key.
+      #
+      # This operation is thread-safe and will return a consistent value
+      # even when called concurrently with write operations.
+      #
+      # @param key [Object] The key to look up
+      # @return [Object, nil] The value associated with the key, or nil if not found
+      #
+      # @example Reading values
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['existing'] = 'value'
+      #   hash['existing']    # => 'value'
+      #   hash['missing']     # => nil
+      #
+      # @example Works with any key type
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash[:symbol] = 'symbol_value'
+      #   hash[42] = 'number_value'
+      #   hash[:symbol]  # => 'symbol_value'
+      #   hash[42]       # => 'number_value'
+      def [](key)
+        @mutex.synchronize { @hash[key] }
+      end
+
+      # Sets the value for the given key.
+      #
+      # This is a thread-safe write operation that ensures data integrity
+      # when called concurrently with other read or write operations.
+      #
+      # @param key [Object] The key to set
+      # @param value [Object] The value to associate with the key
+      # @return [Object] The assigned value
+      #
+      # @example Setting values
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['queue1'] = 'Worker1'
+      #   hash['queue2'] = 'Worker2'
+      #   hash['queue1']  # => 'Worker1'
+      #
+      # @example Overwriting values
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['key'] = 'old_value'
+      #   hash['key'] = 'new_value'
+      #   hash['key']  # => 'new_value'
+      def []=(key, value)
+        @mutex.synchronize { @hash[key] = value }
+      end
+
+      # Removes all key-value pairs from the hash.
+      #
+      # This is a thread-safe write operation that ensures atomicity
+      # when called concurrently with other operations.
+      #
+      # @return [Hash] An empty hash (for compatibility with standard Hash#clear)
+      #
+      # @example Clearing all entries
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['key1'] = 'value1'
+      #   hash['key2'] = 'value2'
+      #   hash.keys.size  # => 2
+      #   hash.clear
+      #   hash.keys.size  # => 0
+      #   hash['key1']    # => nil
+      def clear
+        @mutex.synchronize { @hash.clear }
+      end
+
+      # Returns an array of all keys in the hash.
+      #
+      # This operation is thread-safe and will return a consistent snapshot
+      # of keys even when called concurrently with write operations.
+      #
+      # @return [Array] An array containing all keys in the hash
+      #
+      # @example Getting all keys
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['queue1'] = 'Worker1'
+      #   hash['queue2'] = 'Worker2'
+      #   hash.keys  # => ['queue1', 'queue2'] (order not guaranteed)
+      #
+      # @example Empty hash returns empty array
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash.keys  # => []
+      def keys
+        @mutex.synchronize { @hash.keys }
+      end
+
+      # Returns the value for the given key, or a default value if the key is not found.
+      #
+      # This operation is thread-safe and will return a consistent value
+      # even when called concurrently with write operations.
+      #
+      # @param key [Object] The key to look up
+      # @param default [Object] The value to return if the key is not found
+      # @return [Object] The value associated with the key, or the default value
+      #
+      # @example Fetching with defaults
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash['existing'] = 'found'
+      #   hash.fetch('existing', 'default')  # => 'found'
+      #   hash.fetch('missing', 'default')   # => 'default'
+      #
+      # @example Default parameter is optional
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   hash.fetch('missing')  # => nil
+      #
+      # @example Useful for providing fallback collections
+      #   hash = Shoryuken::Helpers::AtomicHash.new
+      #   workers = hash.fetch('queue_name', [])  # => [] if not found
+      def fetch(key, default = nil)
+        @mutex.synchronize { @hash.fetch(key, default) }
+      end
+    end
+  end
+end

data/lib/shoryuken/helpers/hash_utils.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # Utility methods for hash manipulation.
+    #
+    # This module provides helper methods for common hash operations that were
+    # previously implemented as core class extensions. By using a dedicated
+    # helper module, we avoid polluting the global namespace while maintaining
+    # the same functionality.
+    #
+    # @example Basic usage
+    #   hash = { 'key1' => 'value1', 'key2' => { 'nested' => 'value2' } }
+    #   symbolized = Shoryuken::Helpers::HashUtils.deep_symbolize_keys(hash)
+    #   # => { key1: 'value1', key2: { nested: 'value2' } }
+    module HashUtils
+      class << self
+        # Recursively converts hash keys to symbols.
+        #
+        # This method traverses a hash structure and converts all string keys
+        # to symbols, including nested hashes. Non-hash values are left unchanged.
+        # This is useful for normalizing configuration data loaded from YAML files.
+        #
+        # @param hash [Hash, Object] The hash to convert, or any other object
+        # @return [Hash, Object] Hash with symbolized keys, or the original object if not a hash
+        #
+        # @example Converting a simple hash
+        #   hash = { 'key1' => 'value1', 'key2' => 'value2' }
+        #   HashUtils.deep_symbolize_keys(hash)
+        #   # => { key1: 'value1', key2: 'value2' }
+        #
+        # @example Converting a nested hash
+        #   hash = { 'config' => { 'timeout' => 30, 'retries' => 3 } }
+        #   HashUtils.deep_symbolize_keys(hash)
+        #   # => { config: { timeout: 30, retries: 3 } }
+        #
+        # @example Handling non-hash input gracefully
+        #   HashUtils.deep_symbolize_keys('not a hash')
+        #   # => 'not a hash'
+        #
+        # @example Mixed value types
+        #   hash = { 'string' => 'value', 'number' => 42, 'nested' => { 'bool' => true } }
+        #   HashUtils.deep_symbolize_keys(hash)
+        #   # => { string: 'value', number: 42, nested: { bool: true } }
+        def deep_symbolize_keys(hash)
+          return hash unless hash.is_a?(Hash)
+
+          hash.each_with_object({}) do |(key, value), result|
+            symbol_key = key.is_a?(String) ? key.to_sym : key
+            result[symbol_key] = value.is_a?(Hash) ? deep_symbolize_keys(value) : value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/helpers/string_utils.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # Utility methods for string manipulation.
+    #
+    # This module provides helper methods for common string operations that were
+    # previously implemented as core class extensions. By using a dedicated
+    # helper module, we avoid polluting the global namespace while maintaining
+    # the same functionality.
+    #
+    # @example Basic usage
+    #   klass = Shoryuken::Helpers::StringUtils.constantize('MyWorker')
+    #   # => MyWorker
+    module StringUtils
+      class << self
+        # Converts a string to a constant.
+        #
+        # This method takes a string representation of a constant name and returns
+        # the actual constant. It handles nested constants (e.g., 'Foo::Bar') and
+        # leading double colons (e.g., '::Object'). This is commonly used for
+        # dynamically loading worker classes from configuration.
+        #
+        # @param string [String] The string to convert to a constant
+        # @return [Class, Module] The constant represented by the string
+        # @raise [NameError] if the constant is not found or not defined
+        #
+        # @example Converting a simple class name
+        #   StringUtils.constantize('String')
+        #   # => String
+        #
+        # @example Converting a nested constant
+        #   StringUtils.constantize('Shoryuken::Worker')
+        #   # => Shoryuken::Worker
+        #
+        # @example Handling leading double colon
+        #   StringUtils.constantize('::Object')
+        #   # => Object
+        #
+        # @example Worker class loading
+        #   worker_class = StringUtils.constantize('MyApp::EmailWorker')
+        #   worker_instance = worker_class.new
+        #
+        # @example Error handling
+        #   begin
+        #     StringUtils.constantize('NonExistentClass')
+        #   rescue NameError => e
+        #     puts "Class not found: #{e.message}"
+        #   end
+        def constantize(string)
+          names = string.split('::')
+          names.shift if names.empty? || names.first.empty?
+
+          constant = Object
+
+          names.each do |name|
+            constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+          end
+
+          constant
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/inline_message.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  # A high-performance alternative to OpenStruct for representing SQS messages.
+  #
+  # InlineMessage is a Struct-based implementation that provides the same interface
+  # as the previous OpenStruct-based message representation but with significantly
+  # better performance characteristics. It contains all the essential attributes
+  # needed to represent an Amazon SQS message within the Shoryuken framework.
+  InlineMessage = Struct.new(
+    :body,
+    :attributes,
+    :md5_of_body,
+    :md5_of_message_attributes,
+    :message_attributes,
+    :message_id,
+    :receipt_handle,
+    :delete,
+    :queue_name,
+    keyword_init: true
+  )
+end

data/lib/shoryuken/launcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class Launcher
     include Util

data/lib/shoryuken/logging.rb

@@ -1,12 +1,13 @@
+# frozen_string_literal: true
+
 require 'time'
 require 'logger'
 
 module Shoryuken
   module Logging
-    class Pretty < Logger::Formatter
-      # Provide a call() method that returns the formatted message.
-      def call(severity, time, _program_name, message)
-        "#{time.utc.iso8601} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
+    class Base < ::Logger::Formatter
+      def tid
+        Thread.current['shoryuken_tid'] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
       end
 
       def context
@@ -15,6 +16,19 @@ module Shoryuken
       end
     end
 
+    class Pretty < Base
+      # Provide a call() method that returns the formatted message.
+      def call(severity, time, _program_name, message)
+        "#{time.utc.iso8601} #{Process.pid} TID-#{tid}#{context} #{severity}: #{message}\n"
+      end
+    end
+
+    class WithoutTimestamp < Base
+      def call(severity, _time, _program_name, message)
+        "pid=#{Process.pid} tid=#{tid}#{context} #{severity}: #{message}\n"
+      end
+    end
+
     def self.with_context(msg)
       Thread.current[:shoryuken_context] = msg
       yield
@@ -34,7 +48,7 @@ module Shoryuken
     end
 
     def self.logger=(log)
-      @logger = (log || Logger.new('/dev/null'))
+      @logger = log || Logger.new('/dev/null')
     end
   end
 end

data/lib/shoryuken/manager.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class Manager
     include Util
 
     BATCH_LIMIT = 10
-    # See https://github.com/phstc/shoryuken/issues/348#issuecomment-292847028
+    # See https://github.com/ruby-shoryuken/shoryuken/issues/348#issuecomment-292847028
     MIN_DISPATCH_INTERVAL = 0.1
 
     attr_reader :group
@@ -13,10 +15,10 @@ module Shoryuken
       @fetcher                    = fetcher
       @polling_strategy           = polling_strategy
       @max_processors             = concurrency
-      @busy_processors            = Concurrent::AtomicFixnum.new(0)
+      @busy_processors            = Shoryuken::Helpers::AtomicCounter.new(0)
       @executor                   = executor
-      @running                    = Concurrent::AtomicBoolean.new(true)
-      @stop_new_dispatching       = Concurrent::AtomicBoolean.new(false)
+      @running                    = Shoryuken::Helpers::AtomicBoolean.new(true)
+      @stop_new_dispatching       = Shoryuken::Helpers::AtomicBoolean.new(false)
       @dispatching_release_signal = ::Queue.new
     end
 
@@ -97,7 +99,15 @@ module Shoryuken
       fire_utilization_update_event
 
       Concurrent::Promise
-        .execute(executor: @executor) { Processor.process(queue_name, sqs_msg) }
+        .execute(executor: @executor) do
+          original_priority = Thread.current.priority
+          begin
+            Thread.current.priority = Shoryuken.thread_priority
+            Processor.process(queue_name, sqs_msg)
+          ensure
+            Thread.current.priority = original_priority
+          end
+        end
         .then { processor_done(queue_name) }
         .rescue { processor_done(queue_name) }
     end

data/lib/shoryuken/message.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class Message
     extend Forwardable

data/lib/shoryuken/middleware/chain.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   # Middleware is code configured to run before/after
   # a message is processed.  It is patterned after Rack

data/lib/shoryuken/middleware/server/active_record.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server

data/lib/shoryuken/middleware/server/auto_delete.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server

data/lib/shoryuken/middleware/server/auto_extend_visibility.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server
@@ -29,16 +31,14 @@ module Shoryuken
             queue_visibility_timeout = Shoryuken::Client.queues(queue).visibility_timeout
 
             Concurrent::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
-              begin
-                logger.debug do
-                  "Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
-                end
-
-                sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
-              rescue => ex
-                logger.error do
-                  "Could not auto extend the message #{queue}/#{sqs_msg.message_id} visibility timeout. Error: #{ex.message}"
-                end
+              logger.debug do
+                "Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
+              end
+
+              sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
+            rescue => e
+              logger.error do
+                "Could not auto extend the message #{queue}/#{sqs_msg.message_id} visibility timeout. Error: #{e.message}"
               end
             end
           end

data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server
@@ -14,7 +16,7 @@ module Shoryuken
 
           started_at = Time.now
           yield
-        rescue => ex
+        rescue => e
           retry_intervals = worker.class.get_shoryuken_options['retry_intervals']
 
           if retry_intervals.nil? || !handle_failure(sqs_msg, started_at, retry_intervals)
@@ -23,9 +25,9 @@ module Shoryuken
             raise
           end
 
-          logger.warn { "Message #{sqs_msg.message_id} will attempt retry due to error: #{ex.message}" }
+          logger.warn { "Message #{sqs_msg.message_id} will attempt retry due to error: #{e.message}" }
           # since we didn't raise, lets log the backtrace for debugging purposes.
-          logger.debug { ex.backtrace.join("\n") } unless ex.backtrace.nil?
+          logger.debug { e.backtrace.join("\n") } unless e.backtrace.nil?
         end
 
         private

data/lib/shoryuken/middleware/server/timing.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module Middleware
     module Server

data/lib/shoryuken/options.rb

@@ -1,6 +1,9 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class Options
     DEFAULTS = {
+      thread_priority: -1,
       concurrency: 25,
       queues: [],
       aws: {},
@@ -16,10 +19,12 @@ module Shoryuken
       }
     }.freeze
 
-    attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout, :groups,
-                  :launcher_executor, :reloader, :enable_reloading,
-                  :start_callback, :stop_callback, :worker_executor, :worker_registry, :exception_handlers
-    attr_writer :default_worker_options, :sqs_client
+    attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout,
+                  :groups, :launcher_executor, :reloader, :enable_reloading,
+                  :start_callback, :stop_callback, :worker_executor, :worker_registry,
+                  :exception_handlers
+
+    attr_writer :default_worker_options, :sqs_client, :logger
     attr_reader :sqs_client_receive_message_opts
 
     def initialize
@@ -95,7 +100,11 @@ module Shoryuken
     end
 
     def logger
-      Shoryuken::Logging.logger
+      @logger ||= Shoryuken::Logging.logger
+    end
+
+    def thread_priority
+      @thread_priority ||= options[:thread_priority]
     end
 
     def register_worker(*args)