shoryuken 6.2.1 → 7.0.0.alpha2

data/Gemfile

@@ -5,19 +5,14 @@ gemspec
 
 group :test do
   gem 'activejob'
-  gem 'aws-sdk-core', '~> 3'
-  # Pin to 1.65.0 because of below issues:
-  # - https://github.com/ruby-shoryuken/shoryuken/pull/753#issuecomment-1822720647
-  # - https://github.com/getmoto/moto/issues/7054
-  gem 'aws-sdk-sqs', '1.65.0'
-  gem 'codeclimate-test-reporter', require: nil
   gem 'httparty'
   gem 'multi_xml'
   gem 'simplecov'
+  gem 'warning'
 end
 
 group :development do
   gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
   gem 'pry-byebug'
-  gem 'rubocop', '<= 1.12'
+  gem 'rubocop'
 end

data/README.md

@@ -1,29 +1,25 @@
-**I'm looking for Shoryuken maintainers, are you interested on helping to maintain Shoryuken? [Join our Slack](https://join.slack.com/t/shoryuken/shared_invite/zt-19xjq3iqc-KmoJ6eU6~qvZNqcLzIrjww)**
-
 # Shoryuken
 
-![Shoryuken](shoryuken.jpg)
-
 Shoryuken _sho-ryu-ken_ is a super-efficient [Amazon SQS](https://aws.amazon.com/sqs/) thread-based message processor.
 
 [![Build Status](https://github.com/ruby-shoryuken/shoryuken/workflows/Specs/badge.svg)](https://github.com/ruby-shoryuken/shoryuken/actions)
-[![Code Climate](https://codeclimate.com/github/phstc/shoryuken/badges/gpa.svg)](https://codeclimate.com/github/phstc/shoryuken)
+[![Join the chat at https://slack.shoryuken.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.shoryuken.io)
 
 ## Key features
 
-- [Rails Active Job](https://github.com/phstc/shoryuken/wiki/Rails-Integration-Active-Job)
-- [Queue Load balancing](https://github.com/phstc/shoryuken/wiki/Shoryuken-options#load-balancing)
-- [Concurrency per queue](https://github.com/phstc/shoryuken/wiki/Processing-Groups)
-- [Long Polling](https://github.com/phstc/shoryuken/wiki/Long-Polling)
-- [Batch processing](https://github.com/phstc/shoryuken/wiki/Worker-options#batch)
-- [Auto extend visibility timeout](https://github.com/phstc/shoryuken/wiki/Worker-options#auto_visibility_timeout)
-- [Exponential backoff](https://github.com/phstc/shoryuken/wiki/Worker-options#retry_intervals)
-- [Middleware support](https://github.com/phstc/shoryuken/wiki/Middleware)
+- [Rails Active Job](https://github.com/ruby-shoryuken/shoryuken/wiki/Rails-Integration-Active-Job)
+- [Queue Load balancing](https://github.com/ruby-shoryuken/shoryuken/wiki/Shoryuken-options#load-balancing)
+- [Concurrency per queue](https://github.com/ruby-shoryuken/shoryuken/wiki/Processing-Groups)
+- [Long Polling](https://github.com/ruby-shoryuken/shoryuken/wiki/Long-Polling)
+- [Batch processing](https://github.com/ruby-shoryuken/shoryuken/wiki/Worker-options#batch)
+- [Auto extend visibility timeout](https://github.com/ruby-shoryuken/shoryuken/wiki/Worker-options#auto_visibility_timeout)
+- [Exponential backoff](https://github.com/ruby-shoryuken/shoryuken/wiki/Worker-options#retry_intervals)
+- [Middleware support](https://github.com/ruby-shoryuken/shoryuken/wiki/Middleware)
 - Amazon SQS CLI. See `shoryuken help sqs`
 
 ## Requirements
 
-Ruby 2.4 or greater.
+Ruby 3.0 or greater.
 
 ## Installation
 
@@ -33,14 +29,6 @@ Add this line to your application's Gemfile:
 gem 'shoryuken'
 ```
 
-If you are using AWS SDK version 3, please also add this line:
-
-```ruby
-gem 'aws-sdk-sqs'
-```
-
-The extra gem `aws-sdk-sqs` is required in order to keep Shoryuken compatible with AWS SDK version 2 and 3.
-
 And then execute:
 
 ```shell
@@ -49,11 +37,11 @@ $ bundle
 
 ## Usage
 
-Check the [Getting Started](https://github.com/phstc/shoryuken/wiki/Getting-Started) page.
+Check the [Getting Started](https://github.com/ruby-shoryuken/shoryuken/wiki/Getting-Started) page.
 
 ## More Information
 
-For more information check the [wiki page](https://github.com/phstc/shoryuken/wiki).
+For more information check the [wiki page](https://github.com/ruby-shoryuken/shoryuken/wiki).
 
 ## Credits
 
@@ -61,7 +49,7 @@ For more information check the [wiki page](https://github.com/phstc/shoryuken/wi
 
 ## Contributing
 
-1. Fork it ( https://github.com/phstc/shoryuken/fork )
+1. Fork it ( https://github.com/ruby-shoryuken/shoryuken/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
@@ -69,7 +57,7 @@ For more information check the [wiki page](https://github.com/phstc/shoryuken/wi
 
 ### Testing
 
-To run all unit specs against the latest dependency vesions, execute
+To run all unit specs against the latest dependency versions, execute
 
 ```sh
 bundle exec rake spec

data/Rakefile

@@ -1,4 +1,6 @@
+require 'bundler/setup'
 require 'bundler/gem_tasks'
+
 $stdout.sync = true
 
 begin

data/bin/cli/base.rb

@@ -25,8 +25,7 @@ module Shoryuken
 
         def print_format_column(column, size)
           size_with_padding = size + 4
-          column = column.to_s.ljust(size_with_padding)
-          column
+          column.to_s.ljust(size_with_padding)
         end
 
         def fail_task(msg, quit = true)

data/bin/cli/sqs.rb

@@ -5,7 +5,7 @@ module Shoryuken
   module CLI
     class SQS < Base
       # See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
-      MAX_BATCH_SIZE = 256 * 1024
+      MAX_BATCH_SIZE = 1024 * 1024
 
       namespace :sqs
       class_option :endpoint, aliases: '-e', type: :string, default: ENV['SHORYUKEN_SQS_ENDPOINT'], desc: 'Endpoint URL'
@@ -14,10 +14,17 @@ module Shoryuken
         def normalize_dump_message(message)
           # symbolize_keys is needed for keeping it compatible with `requeue`
           attributes = message[:attributes].symbolize_keys
+
+          # See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_MessageAttributeValue.html
+          # The `string_list_values` and `binary_list_values` are not implemented. Reserved for future use.
+          message_attributes = message[:message_attributes].each_with_object({}) do |(k, v), result|
+            result[k] = v.slice(:data_type, :string_value, :binary_value)
+          end
+
           {
             id: message[:message_id],
             message_body: message[:body],
-            message_attributes: message[:message_attributes],
+            message_attributes: message_attributes,
             message_deduplication_id: attributes[:MessageDeduplicationId],
             message_group_id: attributes[:MessageGroupId]
           }
@@ -104,7 +111,7 @@ module Shoryuken
           attribute_size + body_size
         end
 
-        def find_all(url, limit)
+        def find_all(url, limit, &block)
           count = 0
           batch_size = limit > 10 ? 10 : limit
 
@@ -119,7 +126,7 @@ module Shoryuken
               message_attribute_names: ['All']
             ).messages || []
 
-            messages.each { |m| yield m }
+            messages.each(&block)
 
             count += messages.size
 
@@ -205,7 +212,8 @@ module Shoryuken
       end
 
       desc 'requeue QUEUE-NAME PATH', 'Requeues messages from a dump file'
-      method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'maximum number of messages per batch to send'
+      method_option :batch_size, aliases: '-n', type: :numeric, default: 10,
+                                 desc: 'maximum number of messages per batch to send'
       def requeue(queue_name, path)
         fail_task "Path #{path} not found" unless File.exist?(path)
 

data/bin/shoryuken

@@ -26,7 +26,8 @@ module Shoryuken
       method_option :logfile,     aliases: '-L', type: :string,  desc: 'Path to logfile'
       method_option :pidfile,     aliases: '-P', type: :string,  desc: 'Path to pidfile'
       method_option :verbose,     aliases: '-v', type: :boolean, desc: 'Print more verbose output'
-      method_option :delay,       aliases: '-D', type: :numeric, desc: 'Number of seconds to pause fetching from an empty queue'
+      method_option :delay,       aliases: '-D', type: :numeric,
+                                  desc: 'Number of seconds to pause fetching from an empty queue'
       def start
         opts = options.to_h.symbolize_keys
 

data/docker-compose.yml

@@ -0,0 +1,22 @@
+services:
+  localstack:
+    image: localstack/localstack:latest
+    container_name: localstack
+    ports:
+      - "4566:4566"
+      - "4510-4559:4510-4559"
+    environment:
+      - SERVICES=sqs
+      - DEBUG=${DEBUG:-0}
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - AWS_DEFAULT_REGION=us-east-1
+      - AWS_ACCESS_KEY_ID=test
+      - AWS_SECRET_ACCESS_KEY=test
+    volumes:
+      - "${LOCALSTACK_VOLUME_DIR:-./.localstack}:/var/lib/localstack"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:4566/_localstack/health"]
+      interval: 5s
+      timeout: 10s
+      retries: 5

data/gemfiles/rails_7_0.gemfile

@@ -1,22 +1,19 @@
 # This file was generated by Appraisal
 
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
 group :test do
-  gem "activejob", "~> 7.0"
-  gem "aws-sdk-core", "~> 3"
-  gem "aws-sdk-sqs"
-  gem "codeclimate-test-reporter", require: nil
-  gem "httparty"
-  gem "multi_xml"
-  gem "simplecov"
+  gem 'activejob', '~> 7.0'
+  gem 'httparty'
+  gem 'multi_xml'
+  gem 'simplecov'
+  gem 'warning'
 end
 
 group :development do
-  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
-  gem "rubocop"
-  gem "pry", ">= 0.14.2"
-  gem "pry-byebug", ">= 3.10.1"
+  gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
+  gem 'pry-byebug'
+  gem 'rubocop'
 end
 
-gemspec path: "../"
+gemspec path: '../'

data/gemfiles/rails_7_1.gemfile

@@ -0,0 +1,19 @@
+# This file was generated by Appraisal
+
+source 'https://rubygems.org'
+
+group :test do
+  gem 'activejob', '~> 7.1'
+  gem 'httparty'
+  gem 'multi_xml'
+  gem 'simplecov'
+  gem 'warning'
+end
+
+group :development do
+  gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
+  gem 'pry-byebug'
+  gem 'rubocop'
+end
+
+gemspec path: '../'

data/gemfiles/rails_7_2.gemfile

@@ -0,0 +1,19 @@
+# This file was generated by Appraisal
+
+source 'https://rubygems.org'
+
+group :test do
+  gem 'activejob', '~> 7.2'
+  gem 'httparty'
+  gem 'multi_xml'
+  gem 'simplecov'
+  gem 'warning'
+end
+
+group :development do
+  gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
+  gem 'pry-byebug'
+  gem 'rubocop'
+end
+
+gemspec path: '../'

data/gemfiles/rails_8_0.gemfile

@@ -0,0 +1,19 @@
+# This file was generated by Appraisal
+
+source 'https://rubygems.org'
+
+group :test do
+  gem 'activejob', '~> 8.0'
+  gem 'httparty'
+  gem 'multi_xml'
+  gem 'simplecov'
+  gem 'warning'
+end
+
+group :development do
+  gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
+  gem 'pry-byebug'
+  gem 'rubocop'
+end
+
+gemspec path: '../'

data/lib/shoryuken/body_parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class BodyParser
     class << self
@@ -16,7 +18,7 @@ module Shoryuken
             # JSON.parse
             body_parser.parse(sqs_msg.body)
           elsif body_parser.respond_to?(:load)
-            # see https://github.com/phstc/shoryuken/pull/91
+            # see https://github.com/ruby-shoryuken/shoryuken/pull/91
             # JSON.load
             body_parser.load(sqs_msg.body)
           end

data/lib/shoryuken/client.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class Client
     @@queues = {}

data/lib/shoryuken/default_exception_handler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class DefaultExceptionHandler
     extend Util

data/lib/shoryuken/default_worker_registry.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class DefaultWorkerRegistry < WorkerRegistry
     def initialize
-      @workers = Concurrent::Hash.new
+      @workers = Shoryuken::Helpers::AtomicHash.new
     end
 
     def batch_receive_messages?(queue)
@@ -19,10 +21,10 @@ module Shoryuken
                      message.message_attributes['shoryuken_class'][:string_value]
 
       worker_class = begin
-                       worker_class.constantize
-                     rescue
-                       @workers[queue]
-                     end
+        Shoryuken::Helpers::StringUtils.constantize(worker_class)
+      rescue
+        @workers[queue]
+      end
 
       worker_class.new if worker_class
     end
@@ -32,12 +34,10 @@ module Shoryuken
     end
 
     def register_worker(queue, clazz)
-      if (worker_class = @workers[queue])
-        if worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true
-          fail ArgumentError, "Could not register #{clazz} for #{queue}, "\
-            "because #{worker_class} is already registered for this queue, "\
-            "and Shoryuken doesn't support a batchable worker for a queue with multiple workers"
-        end
+      if (worker_class = @workers[queue]) && (worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true)
+        fail ArgumentError, "Could not register #{clazz} for #{queue}, "\
+          "because #{worker_class} is already registered for this queue, "\
+          "and Shoryuken doesn't support a batchable worker for a queue with multiple workers"
       end
 
       @workers[queue] = clazz

data/lib/shoryuken/environment_loader.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class EnvironmentLoader
     attr_reader :options
@@ -44,7 +46,7 @@ module Shoryuken
       fail ArgumentError, "The supplied config file #{path} does not exist" unless File.exist?(path)
 
       if (result = YAML.load(ERB.new(IO.read(path)).result))
-        result.deep_symbolize_keys
+        Shoryuken::Helpers::HashUtils.deep_symbolize_keys(result)
       else
         {}
       end
@@ -161,11 +163,9 @@ module Shoryuken
       non_existent_queues = []
 
       Shoryuken.ungrouped_queues.uniq.each do |queue|
-        begin
-          Shoryuken::Client.queues(queue)
-        rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue
-          non_existent_queues << queue
-        end
+        Shoryuken::Client.queues(queue)
+      rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue
+        non_existent_queues << queue
       end
 
       return if non_existent_queues.none?

data/lib/shoryuken/extensions/active_job_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # ActiveJob docs: http://edgeguides.rubyonrails.org/active_job_basics.html
 # Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
 
@@ -9,15 +11,15 @@ module ActiveJob
     #
     # Shoryuken ("sho-ryu-ken") is a super-efficient AWS SQS thread based message processor.
     #
-    # Read more about Shoryuken {here}[https://github.com/phstc/shoryuken].
+    # Read more about Shoryuken {here}[https://github.com/ruby-shoryuken/shoryuken].
     #
     # To use Shoryuken set the queue_adapter config to +:shoryuken+.
     #
     #   Rails.application.config.active_job.queue_adapter = :shoryuken
-    class ShoryukenAdapter
+    class ShoryukenAdapter < ActiveJob::QueueAdapters::AbstractAdapter
       class << self
         def instance
-          # https://github.com/phstc/shoryuken/pull/174#issuecomment-174555657
+          # https://github.com/ruby-shoryuken/shoryuken/pull/174#issuecomment-174555657
           @instance ||= new
         end
 
@@ -30,7 +32,12 @@ module ActiveJob
         end
       end
 
-      def enqueue(job, options = {}) #:nodoc:
+      # only required for Rails 7.2.x
+      def enqueue_after_transaction_commit?
+        true
+      end
+
+      def enqueue(job, options = {}) # :nodoc:
         register_worker!(job)
 
         job.sqs_send_message_parameters.merge! options
@@ -41,7 +48,7 @@ module ActiveJob
         queue.send_message send_message_params
       end
 
-      def enqueue_at(job, timestamp) #:nodoc:
+      def enqueue_at(job, timestamp) # :nodoc:
         enqueue(job, delay_seconds: calculate_delay(timestamp))
       end
 
@@ -80,7 +87,7 @@ module ActiveJob
         Shoryuken.register_worker(job.queue_name, JobWrapper)
       end
 
-      class JobWrapper #:nodoc:
+      class JobWrapper # :nodoc:
         include Shoryuken::Worker
 
         shoryuken_options body_parser: :json, auto_delete: true

data/lib/shoryuken/extensions/active_job_concurrent_send_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # ActiveJob docs: http://edgeguides.rubyonrails.org/active_job_basics.html
 # Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
 module ActiveJob
@@ -30,11 +32,9 @@ module ActiveJob
       end
 
       def error_handler
-        @error_handler ||= begin
-          lambda { |error, job, _options|
-            Shoryuken.logger.warn("Failed to enqueue job: #{job.inspect} due to error: #{error}")
-          }
-        end
+        @error_handler ||= lambda { |error, job, _options|
+          Shoryuken.logger.warn("Failed to enqueue job: #{job.inspect} due to error: #{error}")
+        }
       end
 
       private

data/lib/shoryuken/extensions/active_job_extensions.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module ActiveJobExtensions
     # Adds an accessor for SQS SendMessage parameters on ActiveJob jobs

data/lib/shoryuken/fetcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Shoryuken
   class Fetcher
     include Util
@@ -30,13 +32,13 @@ module Shoryuken
 
       begin
         yield
-      rescue => ex
+      rescue => e
         # Tries to auto retry connectivity errors
         raise if attempts >= max_attempts
 
         attempts += 1
 
-        logger.debug { "Retrying fetch attempt #{attempts} for #{ex.message}" }
+        logger.debug { "Retrying fetch attempt #{attempts} for #{e.message}" }
 
         sleep((1..5).to_a.sample)
 

data/lib/shoryuken/helpers/atomic_boolean.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Shoryuken
+  module Helpers
+    # A thread-safe boolean implementation using AtomicCounter as base.
+    # Drop-in replacement for Concurrent::AtomicBoolean without external dependencies.
+    # Uses 1 for true and 0 for false internally.
+    class AtomicBoolean < AtomicCounter
+      # Prevent misuse of counter operations on a boolean
+      undef_method :increment, :decrement
+
+      def initialize(initial_value = false)
+        super(initial_value ? 1 : 0)
+      end
+
+      # Get the current value as boolean
+      def value
+        super != 0
+      end
+
+      # Set the value to true
+      def make_true
+        @mutex.synchronize { @value = 1 }
+        true
+      end
+
+      # Set the value to false
+      def make_false
+        @mutex.synchronize { @value = 0 }
+        false
+      end
+
+      # Check if the value is true
+      def true?
+        @mutex.synchronize { @value != 0 }
+      end
+
+      # Check if the value is false
+      def false?
+        @mutex.synchronize { @value == 0 }
+      end
+    end
+  end
+end