shoryuken 5.0.5 → 5.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2a0f01f40da91819e9e8e0b127a2504fccdd927365866f7971a7df207cea287
-  data.tar.gz: 3da85303fc49af5b0ad06888d667bde14ef12b79c595c98cdc3232d085d08350
+  metadata.gz: 0f2f8c9e8573b699391a74a70d26756e106ccf72b3df9a6fccc32b72d13ec0cd
+  data.tar.gz: 6a009d5a5e571da339d52a08f4ccda19f144ce06132f4a65e95fabc39aeffad4
 SHA512:
-  metadata.gz: 8a94cf02d3c7c25bf8282097449b83f1f4f3b4816b0e7be396048585ccc5667e9dcbdfe839f593c0ed04b2304fc1c0ca2d5d9564c86b854de63de6dafddc5de9
-  data.tar.gz: 20844e414f26d4a6a4e9d3db14bd4ea89bad4550486b91a1a1b8ba6eff2145e89fdf63007e356c707e54dbe8e0d1bb608844ec81db02516e84c4971633fd621d
+  metadata.gz: e359eee09d82a917c4e13541a121c61f78587421a7e5977052552a4f18512eeb9a2eb720ebc8adf323d0a33fd979b37e9c059a95b5b180eda93cab5a3e034f6b
+  data.tar.gz: 5217287d2b7f66125852ac0b824353d934a2a8568e2dc07ca3c98f8991cad1a0dafbda87967a407bc5efaea62b1cafa85708e1ea17ccb659b2743878d079a267

data/.github/workflows/specs.yml

@@ -0,0 +1,62 @@
+name: Specs
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  all_specs:
+    name: All Specs
+    strategy:
+      matrix:
+        ruby: ['2.4', '2.5', '2.6', '2.7', '3.0']
+        gemfile: ['Gemfile', 'gemfiles/aws_sdk_core_2.gemfile']
+    runs-on: ubuntu-20.04
+    services:
+      moto_sqs:
+        image: quay.io/cjlarose/moto-sqs-server:1.1.0
+        ports:
+          - 5000:5000
+    env:
+      BUNDLE_GEMFILE: ${{ matrix.gemfile }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Run specs
+        run: bundle exec rake spec
+      - name: Run integration specs
+        run: bundle exec rake spec:integration
+  rails_specs:
+    name: Rails Specs
+    strategy:
+      matrix:
+        rails: ['4.2', '5.2', '6.0', '6.1']
+        include:
+          - rails: '4.2'
+            ruby: '2.2'
+            gemfile: gemfiles/rails_4_2.gemfile
+          - rails: '5.2'
+            ruby: '2.5'
+            gemfile: gemfiles/rails_5_2.gemfile
+          - rails: '6.0'
+            ruby: '2.6'
+            gemfile: gemfiles/rails_6_0.gemfile
+          - rails: '6.1'
+            ruby: '3.0'
+            gemfile: gemfiles/rails_6_1.gemfile
+    runs-on: ubuntu-20.04
+    env:
+      BUNDLE_GEMFILE: ${{ matrix.gemfile }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Run Rails specs
+        run: bundle exec rake spec:rails

data/.reek.yml

@@ -0,0 +1,5 @@
+---
+detectors:
+
+  UtilityFunction:
+    public_methods_only: true

data/Appraisals

@@ -0,0 +1,36 @@
+appraise 'aws_sdk_core_2' do
+  group :test do
+    gem 'aws-sdk-core', '~> 2'
+    gem 'webrick' # required for Ruby 3
+    remove_gem 'aws-sdk-sqs'
+  end
+end
+
+appraise 'rails_4_2' do
+  group :test do
+    gem 'activejob', '~> 4.2'
+  end
+
+  group :development do
+    gem 'appraisal', '~> 2.2'
+    remove_gem 'pry-byebug'
+  end
+end
+
+appraise 'rails_5_2' do
+  group :test do
+    gem 'activejob', '~> 5.2'
+  end
+end
+
+appraise 'rails_6_0' do
+  group :test do
+    gem 'activejob', '~> 6.0'
+  end
+end
+
+appraise 'rails_6_1' do
+  group :test do
+    gem 'activejob', '~> 6.1'
+  end
+end

data/CHANGELOG.md

@@ -1,3 +1,61 @@
+## [v5.2.3] - 2021-07-29
+
+- Fire new `:utilization_update` event any time a worker pool's utilization changes
+  - [#673](https://github.com/ruby-shoryuken/shoryuken/pull/673)
+
+## [v5.2.2] - 2021-06-22
+
+- When using ActiveJob queue name prefixing, avoid applying prefix to queues configured with a URL or ARN
+  - [#667](https://github.com/ruby-shoryuken/shoryuken/pull/667)
+
+## [v5.2.1] - 2021-04-06
+
+- Reduce message batch sizes in `shoryuken sqs requeue` and `shoryuken sqs mv` commands
+  - [#666](https://github.com/ruby-shoryuken/shoryuken/pull/666)
+
+- Fix bug in `shoryuken sqs requeue` and `shoryuken sqs mv` where those commands would exceed the SQS `SendMessageBatch` maximum payload size
+  - [#663](https://github.com/ruby-shoryuken/shoryuken/issues/663)
+  - [#664](https://github.com/ruby-shoryuken/shoryuken/pull/664)
+
+- Remove test stub for `Concurrent.global_io_executor`
+  - [#662](https://github.com/ruby-shoryuken/shoryuken/pull/662)
+
+- Run integration tests on CI
+  - [#660](https://github.com/ruby-shoryuken/shoryuken/pull/660)
+
+## [v5.2.0] - 2021-02-26
+
+- Set `executions` correctly for ActiveJob jobs
+  - [#657](https://github.com/phstc/shoryuken/pull/657)
+
+## [v5.1.1] - 2021-02-10
+
+- Fix regression in Ruby 3.0 introduced in Shoryuken 5.1.0, where enqueueing jobs with ActiveJob to workers that used keyword arguments would fail
+  - [#654](https://github.com/phstc/shoryuken/pull/654)
+
+## [v5.1.0] - 2021-02-06
+
+- Add support for specifying SQS SendMessage parameters with ActiveJob `.set`
+  - [#635](https://github.com/phstc/shoryuken/pull/635)
+  - [#648](https://github.com/phstc/shoryuken/pull/648)
+  - [#651](https://github.com/phstc/shoryuken/pull/651)
+
+- Unpause FIFO queues on worker completion
+  - [#644](https://github.com/phstc/shoryuken/pull/644)
+
+- Add multiple versions of Rails to test matrix
+  - [#647](https://github.com/phstc/shoryuken/pull/647)
+
+- Migrate from Travis CI to Github Actions
+  - [#649](https://github.com/phstc/shoryuken/pull/649)
+  - [#650](https://github.com/phstc/shoryuken/pull/650)
+  - [#652](https://github.com/phstc/shoryuken/pull/652)
+
+## [v5.0.6] - 2020-12-30
+
+- Load ShoryukenConcurrentSendAdapter when loading Rails
+  - [#642](https://github.com/phstc/shoryuken/pull/642)
+
 ## [v5.0.5] - 2020-06-07
 
 - Add ability to configure queue by ARN

data/Gemfile

@@ -4,7 +4,7 @@ source 'https://rubygems.org'
 gemspec
 
 group :test do
-  gem 'activejob', '~> 4'
+  gem 'activejob'
   gem 'aws-sdk-core', '~> 3'
   gem 'aws-sdk-sqs'
   gem 'codeclimate-test-reporter', require: nil
@@ -14,5 +14,7 @@ group :test do
 end
 
 group :development do
+  gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
+  gem 'pry-byebug', '3.9.0'
   gem 'rubocop'
 end

data/README.md

@@ -1,10 +1,12 @@
+**I'm looking for Shoryuken maintainers, are you interested on helping to maintain Shoryuken? Fill up this form https://forms.gle/8kTso8ixa9Sfp6rJ9**
+
 # Shoryuken
 
 ![Shoryuken](shoryuken.jpg)
 
 Shoryuken _sho-ryu-ken_ is a super-efficient [Amazon SQS](https://aws.amazon.com/sqs/) thread-based message processor.
 
-[![Build Status](https://travis-ci.org/phstc/shoryuken.svg)](https://travis-ci.org/phstc/shoryuken)
+[![Build Status](https://github.com/ruby-shoryuken/shoryuken/workflows/Specs/badge.svg)](https://github.com/ruby-shoryuken/shoryuken/actions)
 [![Code Climate](https://codeclimate.com/github/phstc/shoryuken/badges/gpa.svg)](https://codeclimate.com/github/phstc/shoryuken)
 
 ## Key features
@@ -64,3 +66,23 @@ For more information check the [wiki page](https://github.com/phstc/shoryuken/wi
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create a new Pull Request
+
+### Testing
+
+To run all unit specs against the latest dependency vesions, execute
+
+```sh
+bundle exec rake spec
+```
+
+To run all Rails-related specs against all supported versions of Rails, execute
+
+```sh
+bundle exec appraisal rake spec:rails
+```
+
+To run integration specs, start a mock SQS server on `localhost:5000`. One such option is [cjlarose/moto-sqs-server](https://github.com/cjlarose/moto-sqs-server). Then execute
+
+```sh
+bundle exec rake spec:integration
+```

data/Rakefile

@@ -3,7 +3,21 @@ $stdout.sync = true
 
 begin
   require 'rspec/core/rake_task'
-  RSpec::Core::RakeTask.new(:spec)
+  RSpec::Core::RakeTask.new(:spec) do |t|
+    t.exclude_pattern = 'spec/integration/**/*_spec.rb'
+  end
+
+  namespace :spec do
+    desc 'Run Rails specs only'
+    RSpec::Core::RakeTask.new(:rails) do |t|
+      t.pattern = 'spec/shoryuken/{environment_loader_spec,extensions/active_job_*}.rb'
+    end
+
+    desc 'Run integration specs only'
+    RSpec::Core::RakeTask.new(:integration) do |t|
+      t.pattern = 'spec/integration/**/*_spec.rb'
+    end
+  end
 rescue LoadError
 end
 

data/bin/cli/sqs.rb

@@ -4,6 +4,9 @@ require 'date'
 module Shoryuken
   module CLI
     class SQS < Base
+      # See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
+      MAX_BATCH_SIZE = 256 * 1024
+
       namespace :sqs
       class_option :endpoint, aliases: '-e', type: :string, default: ENV['SHORYUKEN_SQS_ENDPOINT'], desc: 'Endpoint URL'
 
@@ -51,14 +54,56 @@ module Shoryuken
           end
         end
 
-        def batch_send(url, messages, messages_per_batch = 10)
-          messages.to_a.flatten.map(&method(:normalize_dump_message)).each_slice(messages_per_batch) do |batch|
-            sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
-              say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
+        def batch_send(url, messages, max_batch_size = 10)
+          messages = messages.to_a.flatten.map(&method(:normalize_dump_message))
+          batch_send_normalized_messages url, messages, max_batch_size
+        end
+
+        def batch_send_normalized_messages(url, messages, max_batch_size)
+          # Repeatedly take the longest prefix of messages such that
+          # 1. The number of messages is less than or equal to max_batch_size
+          # 2. The total message payload size is less than or equal to the
+          #    batch payload limit
+          while messages.size.positive?
+            batch_size = max_batch_size
+            loop do
+              batch = messages.take batch_size
+
+              unless batch.size == 1 || batch_payload_size(batch) <= MAX_BATCH_SIZE
+                batch_size = batch.size - 1
+                next
+              end
+
+              sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
+                say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
+              end
+              messages = messages.drop batch.size
+              break
             end
           end
         end
 
+        def batch_payload_size(messages)
+          messages.sum(&method(:message_size))
+        end
+
+        def message_size(message)
+          attribute_size = (message[:message_attributes] || []).sum do |name, value|
+            name_size = name.to_s.bytesize
+            data_type_size = value[:data_type].bytesize
+            value_size = if value[:string_value]
+                           value[:string_value].bytesize
+                         elsif value[:binary_value]
+                           value[:binary_value].bytesize
+                         end
+            name_size + data_type_size + value_size
+          end
+
+          body_size = message[:message_body].bytesize
+
+          attribute_size + body_size
+        end
+
         def find_all(url, limit)
           count = 0
           batch_size = limit > 10 ? 10 : limit
@@ -160,7 +205,7 @@ module Shoryuken
       end
 
       desc 'requeue QUEUE-NAME PATH', 'Requeues messages from a dump file'
-      method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'number of messages per batch to send'
+      method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'maximum number of messages per batch to send'
       def requeue(queue_name, path)
         fail_task "Path #{path} not found" unless File.exist?(path)
 

data/gemfiles/.gitignore

@@ -0,0 +1 @@
+*.gemfile.lock

data/gemfiles/aws_sdk_core_2.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob"
+  gem "aws-sdk-core", "~> 2"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+  gem "webrick"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/gemfiles/rails_4_2.gemfile

@@ -0,0 +1,20 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 4.2"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", "~> 2.2"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/gemfiles/rails_5_2.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 5.2"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/gemfiles/rails_6_0.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 6.0"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/gemfiles/rails_6_1.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 6.1"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"