shoryuken 6.2.1 → 7.0.2

data/Gemfile

@@ -5,19 +5,12 @@ gemspec
 
 group :test do
   gem 'activejob'
-  gem 'aws-sdk-core', '~> 3'
-  # Pin to 1.65.0 because of below issues:
-  # - https://github.com/ruby-shoryuken/shoryuken/pull/753#issuecomment-1822720647
-  # - https://github.com/getmoto/moto/issues/7054
-  gem 'aws-sdk-sqs', '1.65.0'
-  gem 'codeclimate-test-reporter', require: nil
   gem 'httparty'
   gem 'multi_xml'
   gem 'simplecov'
+  gem 'warning'
 end
 
 group :development do
-  gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
   gem 'pry-byebug'
-  gem 'rubocop', '<= 1.12'
 end

data/Gemfile.lint

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Code linting
+gem 'rubocop'
+
+# Documentation linting
+gem 'yard-lint'

data/Gemfile.lint.lock

@@ -0,0 +1,69 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.3)
+    json (2.18.0)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
+    parallel (1.27.0)
+    parser (3.3.10.1)
+      ast (~> 2.4.1)
+      racc
+    prism (1.8.0)
+    racc (1.8.1)
+    rainbow (3.1.1)
+    regexp_parser (2.11.3)
+    rubocop (1.82.1)
+      json (~> 2.3)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.48.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.49.0)
+      parser (>= 3.3.7.2)
+      prism (~> 1.7)
+    ruby-progressbar (1.13.0)
+    unicode-display_width (3.2.0)
+      unicode-emoji (~> 4.1)
+    unicode-emoji (4.2.0)
+    yard (0.9.38)
+    yard-lint (1.4.0)
+      yard (~> 0.9)
+      zeitwerk (~> 2.6)
+    zeitwerk (2.7.4)
+
+PLATFORMS
+  ruby
+  x86_64-linux
+
+DEPENDENCIES
+  rubocop
+  yard-lint
+
+CHECKSUMS
+  ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383
+  json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505
+  language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc
+  lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87
+  parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130
+  parser (3.3.10.1) sha256=06f6a725d2cd91e5e7f2b7c32ba143631e1f7c8ae2fb918fc4cebec187e6a688
+  prism (1.8.0) sha256=84453a16ef5530ea62c5f03ec16b52a459575ad4e7b9c2b360fd8ce2c39c1254
+  racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f
+  rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a
+  regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4
+  rubocop (1.82.1) sha256=09f1a6a654a960eda767aebea33e47603080f8e9c9a3f019bf9b94c9cab5e273
+  rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd
+  ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33
+  unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42
+  unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f
+  yard (0.9.38) sha256=721fb82afb10532aa49860655f6cc2eaa7130889df291b052e1e6b268283010f
+  yard-lint (1.4.0) sha256=7dd88fbb08fd77cb840bea899d58812817b36d92291b5693dd0eeb3af9f91f0f
+  zeitwerk (2.7.4) sha256=2bef90f356bdafe9a6c2bd32bcd804f83a4f9b8bc27f3600fff051eb3edcec8b
+
+BUNDLED WITH
+  4.0.3

data/README.md

@@ -1,29 +1,25 @@
-**I'm looking for Shoryuken maintainers, are you interested on helping to maintain Shoryuken? [Join our Slack](https://join.slack.com/t/shoryuken/shared_invite/zt-19xjq3iqc-KmoJ6eU6~qvZNqcLzIrjww)**
-
 # Shoryuken
 
-![Shoryuken](shoryuken.jpg)
-
 Shoryuken _sho-ryu-ken_ is a super-efficient [Amazon SQS](https://aws.amazon.com/sqs/) thread-based message processor.
 
 [![Build Status](https://github.com/ruby-shoryuken/shoryuken/workflows/Specs/badge.svg)](https://github.com/ruby-shoryuken/shoryuken/actions)
-[![Code Climate](https://codeclimate.com/github/phstc/shoryuken/badges/gpa.svg)](https://codeclimate.com/github/phstc/shoryuken)
+[![Join the chat at https://slack.shoryuken.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.shoryuken.io)
 
 ## Key features
 
-- [Rails Active Job](https://github.com/phstc/shoryuken/wiki/Rails-Integration-Active-Job)
-- [Queue Load balancing](https://github.com/phstc/shoryuken/wiki/Shoryuken-options#load-balancing)
-- [Concurrency per queue](https://github.com/phstc/shoryuken/wiki/Processing-Groups)
-- [Long Polling](https://github.com/phstc/shoryuken/wiki/Long-Polling)
-- [Batch processing](https://github.com/phstc/shoryuken/wiki/Worker-options#batch)
-- [Auto extend visibility timeout](https://github.com/phstc/shoryuken/wiki/Worker-options#auto_visibility_timeout)
-- [Exponential backoff](https://github.com/phstc/shoryuken/wiki/Worker-options#retry_intervals)
-- [Middleware support](https://github.com/phstc/shoryuken/wiki/Middleware)
+- [Rails Active Job](https://github.com/ruby-shoryuken/shoryuken/wiki/Rails-Integration-Active-Job)
+- [Queue Load balancing](https://github.com/ruby-shoryuken/shoryuken/wiki/Shoryuken-options#load-balancing)
+- [Concurrency per queue](https://github.com/ruby-shoryuken/shoryuken/wiki/Processing-Groups)
+- [Long Polling](https://github.com/ruby-shoryuken/shoryuken/wiki/Long-Polling)
+- [Batch processing](https://github.com/ruby-shoryuken/shoryuken/wiki/Worker-options#batch)
+- [Auto extend visibility timeout](https://github.com/ruby-shoryuken/shoryuken/wiki/Worker-options#auto_visibility_timeout)
+- [Exponential backoff](https://github.com/ruby-shoryuken/shoryuken/wiki/Worker-options#retry_intervals)
+- [Middleware support](https://github.com/ruby-shoryuken/shoryuken/wiki/Middleware)
 - Amazon SQS CLI. See `shoryuken help sqs`
 
 ## Requirements
 
-Ruby 2.4 or greater.
+Ruby 3.0 or greater.
 
 ## Installation
 
@@ -33,14 +29,6 @@ Add this line to your application's Gemfile:
 gem 'shoryuken'
 ```
 
-If you are using AWS SDK version 3, please also add this line:
-
-```ruby
-gem 'aws-sdk-sqs'
-```
-
-The extra gem `aws-sdk-sqs` is required in order to keep Shoryuken compatible with AWS SDK version 2 and 3.
-
 And then execute:
 
 ```shell
@@ -49,11 +37,11 @@ $ bundle
 
 ## Usage
 
-Check the [Getting Started](https://github.com/phstc/shoryuken/wiki/Getting-Started) page.
+Check the [Getting Started](https://github.com/ruby-shoryuken/shoryuken/wiki/Getting-Started) page.
 
 ## More Information
 
-For more information check the [wiki page](https://github.com/phstc/shoryuken/wiki).
+For more information check the [wiki page](https://github.com/ruby-shoryuken/shoryuken/wiki).
 
 ## Credits
 
@@ -61,7 +49,7 @@ For more information check the [wiki page](https://github.com/phstc/shoryuken/wi
 
 ## Contributing
 
-1. Fork it ( https://github.com/phstc/shoryuken/fork )
+1. Fork it ( https://github.com/ruby-shoryuken/shoryuken/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
@@ -69,21 +57,16 @@ For more information check the [wiki page](https://github.com/phstc/shoryuken/wi
 
 ### Testing
 
-To run all unit specs against the latest dependency vesions, execute
+To run all unit specs against the latest dependency versions, execute
 
 ```sh
 bundle exec rake spec
 ```
 
-To run all Rails-related specs against all supported versions of Rails, execute
-
-```sh
-bundle exec appraisal rake spec:rails
-```
-
-To run integration specs, start a mock SQS server on `localhost:5000`. One such option is [cjlarose/moto-sqs-server](https://github.com/cjlarose/moto-sqs-server). Then execute
+To run integration specs (including Rails tests), start ElasticMQ and run:
 
 ```sh
+docker compose up -d
 bundle exec rake spec:integration
 ```
 

data/Rakefile

@@ -1,21 +1,17 @@
+require 'bundler/setup'
 require 'bundler/gem_tasks'
+
 $stdout.sync = true
 
 begin
   require 'rspec/core/rake_task'
-  RSpec::Core::RakeTask.new(:spec) do |t|
-    t.exclude_pattern = 'spec/integration/**/*_spec.rb'
-  end
+  RSpec::Core::RakeTask.new(:spec)
 
   namespace :spec do
-    desc 'Run Rails specs only'
-    RSpec::Core::RakeTask.new(:rails) do |t|
-      t.pattern = 'spec/shoryuken/{environment_loader_spec,extensions/active_job_*}.rb'
-    end
-
     desc 'Run integration specs only'
-    RSpec::Core::RakeTask.new(:integration) do |t|
-      t.pattern = 'spec/integration/**/*_spec.rb'
+    task :integration do
+      puts "Running integration tests..."
+      system('./bin/integrations') || exit(1)
     end
   end
 rescue LoadError

data/bin/clean_sqs

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Removes all integration test SQS queues from ElasticMQ
+#
+# Useful when having a long-running ElasticMQ instance that cannot be fully
+# restarted between test runs. All integration test queues use the 'it-' prefix,
+# making them easy to identify and remove.
+#
+# Usage:
+#   bin/clean_sqs
+
+require 'aws-sdk-sqs'
+
+THREADS_COUNT = 3
+
+sqs = Aws::SQS::Client.new(
+  region: 'us-east-1',
+  endpoint: 'http://localhost:9324',
+  access_key_id: 'fake',
+  secret_access_key: 'fake'
+)
+
+# Find all queues with 'it-' prefix
+response = sqs.list_queues(queue_name_prefix: 'it-')
+queues_for_removal = response.queue_urls || []
+
+if queues_for_removal.empty?
+  puts "No integration test queues found (prefix: it-)"
+  exit 0
+end
+
+puts "Found #{queues_for_removal.size} queues to remove"
+
+queue = SizedQueue.new(THREADS_COUNT)
+
+threads = Array.new(THREADS_COUNT) do
+  Thread.new do
+    while (queue_url = queue.pop)
+      queue_name = queue_url.split('/').last
+      puts "Removing queue: #{queue_name}"
+      sqs.delete_queue(queue_url: queue_url)
+    end
+  end
+end
+
+queues_for_removal.each { |url| queue << url }
+
+queue.close
+threads.each(&:join)
+
+puts "Cleanup complete"

data/bin/cli/base.rb

@@ -1,7 +1,14 @@
+# frozen_string_literal: true
+
 module Shoryuken
   module CLI
+    # Base class for CLI commands providing common helper methods.
     class Base < Thor
       no_commands do
+        # Prints entries as a formatted table
+        #
+        # @param entries [Array<Array>] rows to print as a table
+        # @return [void]
         def print_table(entries)
           column_sizes = print_columns_size(entries)
 
@@ -10,6 +17,10 @@ module Shoryuken
           end
         end
 
+        # Calculates the maximum width for each column
+        #
+        # @param entries [Array<Array>] the table rows
+        # @return [Hash<Integer, Integer>] column index to max width mapping
         def print_columns_size(entries)
           column_sizes = Hash.new(0)
 
@@ -23,12 +34,21 @@ module Shoryuken
           column_sizes
         end
 
+        # Formats a column value with padding
+        #
+        # @param column [Object] the column value to format
+        # @param size [Integer] the target width
+        # @return [String] the formatted column
         def print_format_column(column, size)
           size_with_padding = size + 4
-          column = column.to_s.ljust(size_with_padding)
-          column
+          column.to_s.ljust(size_with_padding)
         end
 
+        # Outputs a failure message and optionally exits
+        #
+        # @param msg [String] the failure message
+        # @param quit [Boolean] whether to exit the program
+        # @return [void]
         def fail_task(msg, quit = true)
           say "[FAIL] #{msg}", :red
           exit(1) if quit

data/bin/cli/sqs.rb

@@ -1,28 +1,47 @@
+# frozen_string_literal: true
+
 require 'date'
 
 # rubocop:disable Metrics/BlockLength
 module Shoryuken
   module CLI
+    # SQS command line interface for queue management operations.
+    # Provides commands for listing, creating, deleting, moving, and dumping queue messages.
     class SQS < Base
-      # See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
-      MAX_BATCH_SIZE = 256 * 1024
+      # Maximum batch size in bytes for SQS batch operations
+      # @see https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
+      MAX_BATCH_SIZE = 1024 * 1024
 
       namespace :sqs
       class_option :endpoint, aliases: '-e', type: :string, default: ENV['SHORYUKEN_SQS_ENDPOINT'], desc: 'Endpoint URL'
 
       no_commands do
+        # Normalizes a dump message for requeuing
+        #
+        # @param message [Hash] the dumped message hash
+        # @return [Hash] the normalized message
         def normalize_dump_message(message)
           # symbolize_keys is needed for keeping it compatible with `requeue`
-          attributes = message[:attributes].symbolize_keys
+          attributes = message[:attributes].transform_keys(&:to_sym)
+
+          # See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_MessageAttributeValue.html
+          # The `string_list_values` and `binary_list_values` are not implemented. Reserved for future use.
+          message_attributes = message[:message_attributes].each_with_object({}) do |(k, v), result|
+            result[k] = v.slice(:data_type, :string_value, :binary_value)
+          end
+
           {
             id: message[:message_id],
             message_body: message[:body],
-            message_attributes: message[:message_attributes],
+            message_attributes: message_attributes,
             message_deduplication_id: attributes[:MessageDeduplicationId],
             message_group_id: attributes[:MessageGroupId]
           }
         end
 
+        # Returns the SQS client options based on CLI options
+        #
+        # @return [Hash] the client options hash
         def client_options
           endpoint = options[:endpoint]
           {}.tap do |hash|
@@ -30,16 +49,28 @@ module Shoryuken
           end
         end
 
+        # Returns the SQS client instance
+        #
+        # @return [Aws::SQS::Client] the SQS client
         def sqs
           @_sqs ||= Aws::SQS::Client.new(client_options)
         end
 
+        # Finds the URL for a queue by name
+        #
+        # @param queue_name [String] the queue name
+        # @return [String] the queue URL
         def find_queue_url(queue_name)
           sqs.get_queue_url(queue_name: queue_name).queue_url
         rescue Aws::SQS::Errors::NonExistentQueue
           fail_task "The specified queue #{queue_name} does not exist"
         end
 
+        # Batch deletes messages from a queue
+        #
+        # @param url [String] the queue URL
+        # @param messages [Array<Aws::SQS::Types::Message>] the messages to delete
+        # @return [void]
         def batch_delete(url, messages)
           messages.to_a.flatten.each_slice(10) do |batch|
             sqs.delete_message_batch(
@@ -54,11 +85,23 @@ module Shoryuken
           end
         end
 
+        # Batch sends messages to a queue
+        #
+        # @param url [String] the queue URL
+        # @param messages [Array<Hash>] the messages to send
+        # @param max_batch_size [Integer] maximum messages per batch
+        # @return [void]
         def batch_send(url, messages, max_batch_size = 10)
           messages = messages.to_a.flatten.map(&method(:normalize_dump_message))
           batch_send_normalized_messages url, messages, max_batch_size
         end
 
+        # Batch sends normalized messages to a queue
+        #
+        # @param url [String] the queue URL
+        # @param messages [Array<Hash>] the normalized messages
+        # @param max_batch_size [Integer] maximum messages per batch
+        # @return [void]
         def batch_send_normalized_messages(url, messages, max_batch_size)
           # Repeatedly take the longest prefix of messages such that
           # 1. The number of messages is less than or equal to max_batch_size
@@ -83,10 +126,18 @@ module Shoryuken
           end
         end
 
+        # Calculates the total payload size of a batch
+        #
+        # @param messages [Array<Hash>] the messages
+        # @return [Integer] total size in bytes
         def batch_payload_size(messages)
           messages.sum(&method(:message_size))
         end
 
+        # Calculates the size of a single message
+        #
+        # @param message [Hash] the message
+        # @return [Integer] size in bytes
         def message_size(message)
           attribute_size = (message[:message_attributes] || []).sum do |name, value|
             name_size = name.to_s.bytesize
@@ -104,7 +155,13 @@ module Shoryuken
           attribute_size + body_size
         end
 
-        def find_all(url, limit)
+        # Receives all messages from a queue up to a limit
+        #
+        # @param url [String] the queue URL
+        # @param limit [Integer, Float] maximum messages to receive
+        # @yield [Aws::SQS::Types::Message] yields each received message
+        # @return [Integer] the number of messages received
+        def find_all(url, limit, &block)
           count = 0
           batch_size = limit > 10 ? 10 : limit
 
@@ -119,7 +176,7 @@ module Shoryuken
               message_attribute_names: ['All']
             ).messages || []
 
-            messages.each { |m| yield m }
+            messages.each(&block)
 
             count += messages.size
 
@@ -130,6 +187,10 @@ module Shoryuken
           count
         end
 
+        # Lists queues and prints their attributes as a table
+        #
+        # @param urls [Array<String>] the queue URLs
+        # @return [void]
         def list_and_print_queues(urls)
           attrs = %w[QueueArn ApproximateNumberOfMessages ApproximateNumberOfMessagesNotVisible LastModifiedTimestamp]
 
@@ -147,6 +208,11 @@ module Shoryuken
           print_table(entries)
         end
 
+        # Generates the dump file path for a queue
+        #
+        # @param path [String] the directory path
+        # @param queue_name [String] the queue name
+        # @return [String] the full file path
         def dump_file(path, queue_name)
           File.join(path, "#{queue_name}-#{Date.today}.jsonl")
         end
@@ -205,7 +271,8 @@ module Shoryuken
       end
 
       desc 'requeue QUEUE-NAME PATH', 'Requeues messages from a dump file'
-      method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'maximum number of messages per batch to send'
+      method_option :batch_size, aliases: '-n', type: :numeric, default: 10,
+                                 desc: 'maximum number of messages per batch to send'
       def requeue(queue_name, path)
         fail_task "Path #{path} not found" unless File.exist?(path)