shoryuken 7.0.0.alpha1 → 7.0.0.alpha2

data/Gemfile

@@ -8,6 +8,7 @@ group :test do
   gem 'httparty'
   gem 'multi_xml'
   gem 'simplecov'
+  gem 'warning'
 end
 
 group :development do

data/README.md

@@ -1,21 +1,20 @@
 # Shoryuken
 
-![Shoryuken](shoryuken.jpg)
-
 Shoryuken _sho-ryu-ken_ is a super-efficient [Amazon SQS](https://aws.amazon.com/sqs/) thread-based message processor.
 
 [![Build Status](https://github.com/ruby-shoryuken/shoryuken/workflows/Specs/badge.svg)](https://github.com/ruby-shoryuken/shoryuken/actions)
+[![Join the chat at https://slack.shoryuken.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.shoryuken.io)
 
 ## Key features
 
-- [Rails Active Job](https://github.com/phstc/shoryuken/wiki/Rails-Integration-Active-Job)
-- [Queue Load balancing](https://github.com/phstc/shoryuken/wiki/Shoryuken-options#load-balancing)
-- [Concurrency per queue](https://github.com/phstc/shoryuken/wiki/Processing-Groups)
-- [Long Polling](https://github.com/phstc/shoryuken/wiki/Long-Polling)
-- [Batch processing](https://github.com/phstc/shoryuken/wiki/Worker-options#batch)
-- [Auto extend visibility timeout](https://github.com/phstc/shoryuken/wiki/Worker-options#auto_visibility_timeout)
-- [Exponential backoff](https://github.com/phstc/shoryuken/wiki/Worker-options#retry_intervals)
-- [Middleware support](https://github.com/phstc/shoryuken/wiki/Middleware)
+- [Rails Active Job](https://github.com/ruby-shoryuken/shoryuken/wiki/Rails-Integration-Active-Job)
+- [Queue Load balancing](https://github.com/ruby-shoryuken/shoryuken/wiki/Shoryuken-options#load-balancing)
+- [Concurrency per queue](https://github.com/ruby-shoryuken/shoryuken/wiki/Processing-Groups)
+- [Long Polling](https://github.com/ruby-shoryuken/shoryuken/wiki/Long-Polling)
+- [Batch processing](https://github.com/ruby-shoryuken/shoryuken/wiki/Worker-options#batch)
+- [Auto extend visibility timeout](https://github.com/ruby-shoryuken/shoryuken/wiki/Worker-options#auto_visibility_timeout)
+- [Exponential backoff](https://github.com/ruby-shoryuken/shoryuken/wiki/Worker-options#retry_intervals)
+- [Middleware support](https://github.com/ruby-shoryuken/shoryuken/wiki/Middleware)
 - Amazon SQS CLI. See `shoryuken help sqs`
 
 ## Requirements
@@ -38,11 +37,11 @@ $ bundle
 
 ## Usage
 
-Check the [Getting Started](https://github.com/phstc/shoryuken/wiki/Getting-Started) page.
+Check the [Getting Started](https://github.com/ruby-shoryuken/shoryuken/wiki/Getting-Started) page.
 
 ## More Information
 
-For more information check the [wiki page](https://github.com/phstc/shoryuken/wiki).
+For more information check the [wiki page](https://github.com/ruby-shoryuken/shoryuken/wiki).
 
 ## Credits
 
@@ -50,7 +49,7 @@ For more information check the [wiki page](https://github.com/phstc/shoryuken/wi
 
 ## Contributing
 
-1. Fork it ( https://github.com/phstc/shoryuken/fork )
+1. Fork it ( https://github.com/ruby-shoryuken/shoryuken/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)

data/bin/cli/base.rb

@@ -25,8 +25,7 @@ module Shoryuken
 
         def print_format_column(column, size)
           size_with_padding = size + 4
-          column = column.to_s.ljust(size_with_padding)
-          column
+          column.to_s.ljust(size_with_padding)
         end
 
         def fail_task(msg, quit = true)

data/bin/cli/sqs.rb

@@ -5,7 +5,7 @@ module Shoryuken
   module CLI
     class SQS < Base
       # See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
-      MAX_BATCH_SIZE = 256 * 1024
+      MAX_BATCH_SIZE = 1024 * 1024
 
       namespace :sqs
       class_option :endpoint, aliases: '-e', type: :string, default: ENV['SHORYUKEN_SQS_ENDPOINT'], desc: 'Endpoint URL'
@@ -111,7 +111,7 @@ module Shoryuken
           attribute_size + body_size
         end
 
-        def find_all(url, limit)
+        def find_all(url, limit, &block)
           count = 0
           batch_size = limit > 10 ? 10 : limit
 
@@ -126,7 +126,7 @@ module Shoryuken
               message_attribute_names: ['All']
             ).messages || []
 
-            messages.each { |m| yield m }
+            messages.each(&block)
 
             count += messages.size
 
@@ -212,7 +212,8 @@ module Shoryuken
       end
 
       desc 'requeue QUEUE-NAME PATH', 'Requeues messages from a dump file'
-      method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'maximum number of messages per batch to send'
+      method_option :batch_size, aliases: '-n', type: :numeric, default: 10,
+                                 desc: 'maximum number of messages per batch to send'
       def requeue(queue_name, path)
         fail_task "Path #{path} not found" unless File.exist?(path)
 

data/bin/shoryuken

@@ -26,7 +26,8 @@ module Shoryuken
       method_option :logfile,     aliases: '-L', type: :string,  desc: 'Path to logfile'
       method_option :pidfile,     aliases: '-P', type: :string,  desc: 'Path to pidfile'
       method_option :verbose,     aliases: '-v', type: :boolean, desc: 'Print more verbose output'
-      method_option :delay,       aliases: '-D', type: :numeric, desc: 'Number of seconds to pause fetching from an empty queue'
+      method_option :delay,       aliases: '-D', type: :numeric,
+                                  desc: 'Number of seconds to pause fetching from an empty queue'
       def start
         opts = options.to_h.symbolize_keys
 

data/gemfiles/rails_7_0.gemfile

@@ -1,19 +1,19 @@
 # This file was generated by Appraisal
 
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
 group :test do
-  gem "activejob", "~> 7.0"
-  gem "httparty"
-  gem "multi_xml"
-  gem "simplecov"
+  gem 'activejob', '~> 7.0'
+  gem 'httparty'
+  gem 'multi_xml'
+  gem 'simplecov'
+  gem 'warning'
 end
 
 group :development do
-  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
-  gem "rubocop"
-  gem "pry", ">= 0.14.2"
-  gem "pry-byebug", ">= 3.10.1"
+  gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
+  gem 'pry-byebug'
+  gem 'rubocop'
 end
 
-gemspec path: "../"
+gemspec path: '../'

data/gemfiles/rails_7_1.gemfile

@@ -1,18 +1,19 @@
 # This file was generated by Appraisal
 
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
 group :test do
-  gem "activejob", "~> 7.1"
-  gem "httparty"
-  gem "multi_xml"
-  gem "simplecov"
+  gem 'activejob', '~> 7.1'
+  gem 'httparty'
+  gem 'multi_xml'
+  gem 'simplecov'
+  gem 'warning'
 end
 
 group :development do
-  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
-  gem "pry-byebug"
-  gem "rubocop"
+  gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
+  gem 'pry-byebug'
+  gem 'rubocop'
 end
 
-gemspec path: "../"
+gemspec path: '../'

data/gemfiles/rails_7_2.gemfile

@@ -1,18 +1,19 @@
 # This file was generated by Appraisal
 
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
 group :test do
-  gem "activejob", "~> 7.2"
-  gem "httparty"
-  gem "multi_xml"
-  gem "simplecov"
+  gem 'activejob', '~> 7.2'
+  gem 'httparty'
+  gem 'multi_xml'
+  gem 'simplecov'
+  gem 'warning'
 end
 
 group :development do
-  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
-  gem "pry-byebug"
-  gem "rubocop"
+  gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
+  gem 'pry-byebug'
+  gem 'rubocop'
 end
 
-gemspec path: "../"
+gemspec path: '../'

data/gemfiles/rails_8_0.gemfile

@@ -1,18 +1,19 @@
 # This file was generated by Appraisal
 
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
 group :test do
-  gem "activejob", "~> 8.0"
-  gem "httparty"
-  gem "multi_xml"
-  gem "simplecov"
+  gem 'activejob', '~> 8.0'
+  gem 'httparty'
+  gem 'multi_xml'
+  gem 'simplecov'
+  gem 'warning'
 end
 
 group :development do
-  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
-  gem "pry-byebug"
-  gem "rubocop"
+  gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
+  gem 'pry-byebug'
+  gem 'rubocop'
 end
 
-gemspec path: "../"
+gemspec path: '../'

data/lib/shoryuken/body_parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class BodyParser
     class << self
@@ -16,7 +18,7 @@ module Shoryuken
             # JSON.parse
             body_parser.parse(sqs_msg.body)
           elsif body_parser.respond_to?(:load)
-            # see https://github.com/phstc/shoryuken/pull/91
+            # see https://github.com/ruby-shoryuken/shoryuken/pull/91
             # JSON.load
             body_parser.load(sqs_msg.body)
           end

data/lib/shoryuken/client.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class Client
     @@queues = {}

data/lib/shoryuken/default_exception_handler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class DefaultExceptionHandler
     extend Util

data/lib/shoryuken/default_worker_registry.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class DefaultWorkerRegistry < WorkerRegistry
     def initialize
-      @workers = Concurrent::Hash.new
+      @workers = Shoryuken::Helpers::AtomicHash.new
     end
 
     def batch_receive_messages?(queue)
@@ -19,10 +21,10 @@ module Shoryuken
                      message.message_attributes['shoryuken_class'][:string_value]
 
       worker_class = begin
-                       worker_class.constantize
-                     rescue
-                       @workers[queue]
-                     end
+        Shoryuken::Helpers::StringUtils.constantize(worker_class)
+      rescue
+        @workers[queue]
+      end
 
       worker_class.new if worker_class
     end
@@ -32,12 +34,10 @@ module Shoryuken
     end
 
     def register_worker(queue, clazz)
-      if (worker_class = @workers[queue])
-        if worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true
-          fail ArgumentError, "Could not register #{clazz} for #{queue}, "\
-            "because #{worker_class} is already registered for this queue, "\
-            "and Shoryuken doesn't support a batchable worker for a queue with multiple workers"
-        end
+      if (worker_class = @workers[queue]) && (worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true)
+        fail ArgumentError, "Could not register #{clazz} for #{queue}, "\
+          "because #{worker_class} is already registered for this queue, "\
+          "and Shoryuken doesn't support a batchable worker for a queue with multiple workers"
       end
 
       @workers[queue] = clazz

data/lib/shoryuken/environment_loader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class EnvironmentLoader
     attr_reader :options
@@ -44,7 +46,7 @@ module Shoryuken
       fail ArgumentError, "The supplied config file #{path} does not exist" unless File.exist?(path)
 
       if (result = YAML.load(ERB.new(IO.read(path)).result))
-        result.deep_symbolize_keys
+        Shoryuken::Helpers::HashUtils.deep_symbolize_keys(result)
       else
         {}
       end
@@ -161,11 +163,9 @@ module Shoryuken
       non_existent_queues = []
 
       Shoryuken.ungrouped_queues.uniq.each do |queue|
-        begin
-          Shoryuken::Client.queues(queue)
-        rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue
-          non_existent_queues << queue
-        end
+        Shoryuken::Client.queues(queue)
+      rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue
+        non_existent_queues << queue
       end
 
       return if non_existent_queues.none?

data/lib/shoryuken/extensions/active_job_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # ActiveJob docs: http://edgeguides.rubyonrails.org/active_job_basics.html
 # Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
 
@@ -9,15 +11,15 @@ module ActiveJob
     #
     # Shoryuken ("sho-ryu-ken") is a super-efficient AWS SQS thread based message processor.
     #
-    # Read more about Shoryuken {here}[https://github.com/phstc/shoryuken].
+    # Read more about Shoryuken {here}[https://github.com/ruby-shoryuken/shoryuken].
     #
     # To use Shoryuken set the queue_adapter config to +:shoryuken+.
     #
     #   Rails.application.config.active_job.queue_adapter = :shoryuken
-    class ShoryukenAdapter
+    class ShoryukenAdapter < ActiveJob::QueueAdapters::AbstractAdapter
       class << self
         def instance
-          # https://github.com/phstc/shoryuken/pull/174#issuecomment-174555657
+          # https://github.com/ruby-shoryuken/shoryuken/pull/174#issuecomment-174555657
           @instance ||= new
         end
 
@@ -35,7 +37,7 @@ module ActiveJob
         true
       end
 
-      def enqueue(job, options = {}) #:nodoc:
+      def enqueue(job, options = {}) # :nodoc:
         register_worker!(job)
 
         job.sqs_send_message_parameters.merge! options
@@ -46,7 +48,7 @@ module ActiveJob
         queue.send_message send_message_params
       end
 
-      def enqueue_at(job, timestamp) #:nodoc:
+      def enqueue_at(job, timestamp) # :nodoc:
         enqueue(job, delay_seconds: calculate_delay(timestamp))
       end
 
@@ -85,7 +87,7 @@ module ActiveJob
         Shoryuken.register_worker(job.queue_name, JobWrapper)
       end
 
-      class JobWrapper #:nodoc:
+      class JobWrapper # :nodoc:
         include Shoryuken::Worker
 
         shoryuken_options body_parser: :json, auto_delete: true

data/lib/shoryuken/extensions/active_job_concurrent_send_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # ActiveJob docs: http://edgeguides.rubyonrails.org/active_job_basics.html
 # Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
 module ActiveJob
@@ -30,11 +32,9 @@ module ActiveJob
       end
 
       def error_handler
-        @error_handler ||= begin
-          lambda { |error, job, _options|
-            Shoryuken.logger.warn("Failed to enqueue job: #{job.inspect} due to error: #{error}")
-          }
-        end
+        @error_handler ||= lambda { |error, job, _options|
+          Shoryuken.logger.warn("Failed to enqueue job: #{job.inspect} due to error: #{error}")
+        }
       end
 
       private

data/lib/shoryuken/extensions/active_job_extensions.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module ActiveJobExtensions
     # Adds an accessor for SQS SendMessage parameters on ActiveJob jobs

data/lib/shoryuken/fetcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class Fetcher
     include Util
@@ -30,13 +32,13 @@ module Shoryuken
 
       begin
         yield
-      rescue => ex
+      rescue => e
         # Tries to auto retry connectivity errors
         raise if attempts >= max_attempts
 
         attempts += 1
 
-        logger.debug { "Retrying fetch attempt #{attempts} for #{ex.message}" }
+        logger.debug { "Retrying fetch attempt #{attempts} for #{e.message}" }
 
         sleep((1..5).to_a.sample)
 

data/lib/shoryuken/helpers/atomic_boolean.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # A thread-safe boolean implementation using AtomicCounter as base.
+    # Drop-in replacement for Concurrent::AtomicBoolean without external dependencies.
+    # Uses 1 for true and 0 for false internally.
+    class AtomicBoolean < AtomicCounter
+      # Prevent misuse of counter operations on a boolean
+      undef_method :increment, :decrement
+
+      def initialize(initial_value = false)
+        super(initial_value ? 1 : 0)
+      end
+
+      # Get the current value as boolean
+      def value
+        super != 0
+      end
+
+      # Set the value to true
+      def make_true
+        @mutex.synchronize { @value = 1 }
+        true
+      end
+
+      # Set the value to false
+      def make_false
+        @mutex.synchronize { @value = 0 }
+        false
+      end
+
+      # Check if the value is true
+      def true?
+        @mutex.synchronize { @value != 0 }
+      end
+
+      # Check if the value is false
+      def false?
+        @mutex.synchronize { @value == 0 }
+      end
+    end
+  end
+end

data/lib/shoryuken/helpers/atomic_counter.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # A thread-safe counter implementation using Ruby's Mutex.
+    #
+    # This class provides atomic operations for incrementing, decrementing, and reading
+    # integer values in a thread-safe manner. It serves as a drop-in replacement for
+    # Concurrent::AtomicFixnum without requiring external dependencies.
+    #
+    # The implementation uses a Mutex to ensure thread safety across all Ruby
+    # implementations including JRuby, where true parallelism makes atomic operations
+    # critical for data integrity.
+    #
+    # @note This class is optimized for scenarios with frequent atomic updates
+    #   and occasional reads, such as tracking active worker counts.
+    #
+    # @example Basic usage
+    #   counter = Shoryuken::Helpers::AtomicCounter.new(0)
+    #   counter.increment  # => 1
+    #   counter.increment  # => 2
+    #   counter.value      # => 2
+    #   counter.decrement  # => 1
+    #
+    # @example Tracking busy processors
+    #   @busy_processors = Shoryuken::Helpers::AtomicCounter.new(0)
+    #
+    #   # When starting work
+    #   @busy_processors.increment
+    #
+    #   # When work is done
+    #   @busy_processors.decrement
+    #
+    #   # Check current load
+    #   current_busy = @busy_processors.value
+    class AtomicCounter
+      # Creates a new atomic counter with the specified initial value.
+      #
+      # @param initial_value [Integer] The starting value for the counter
+      # @return [AtomicCounter] A new atomic counter instance
+      #
+      # @example Create counter starting at zero
+      #   counter = Shoryuken::Helpers::AtomicCounter.new
+      #   counter.value  # => 0
+      #
+      # @example Create counter with custom initial value
+      #   counter = Shoryuken::Helpers::AtomicCounter.new(100)
+      #   counter.value  # => 100
+      def initialize(initial_value = 0)
+        @mutex = Mutex.new
+        @value = initial_value
+      end
+
+      # Returns the current value of the counter.
+      #
+      # This operation is thread-safe and will return a consistent value
+      # even when called concurrently with increment/decrement operations.
+      #
+      # @return [Integer] The current counter value
+      #
+      # @example Reading the current value
+      #   counter = Shoryuken::Helpers::AtomicCounter.new(42)
+      #   counter.value  # => 42
+      def value
+        @mutex.synchronize { @value }
+      end
+
+      # Atomically increments the counter by 1 and returns the new value.
+      #
+      # This operation is thread-safe and can be called concurrently from
+      # multiple threads without risk of data corruption or lost updates.
+      #
+      # @return [Integer] The new counter value after incrementing
+      #
+      # @example Incrementing the counter
+      #   counter = Shoryuken::Helpers::AtomicCounter.new(5)
+      #   counter.increment  # => 6
+      #   counter.increment  # => 7
+      def increment
+        @mutex.synchronize { @value += 1 }
+      end
+
+      # Atomically decrements the counter by 1 and returns the new value.
+      #
+      # This operation is thread-safe and can be called concurrently from
+      # multiple threads without risk of data corruption or lost updates.
+      # The counter can go negative if decremented below zero.
+      #
+      # @return [Integer] The new counter value after decrementing
+      #
+      # @example Decrementing the counter
+      #   counter = Shoryuken::Helpers::AtomicCounter.new(5)
+      #   counter.decrement  # => 4
+      #   counter.decrement  # => 3
+      #
+      # @example Counter can go negative
+      #   counter = Shoryuken::Helpers::AtomicCounter.new(0)
+      #   counter.decrement  # => -1
+      def decrement
+        @mutex.synchronize { @value -= 1 }
+      end
+    end
+  end
+end