shoryuken 5.0.4 → 5.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9793af33c7faa6fed94f9ba3c4d195de11fbb2bac9147740cec68f3f0a9b60b0
-  data.tar.gz: ba16095e67066c94e72fb9472d1f34d238bfbde6c1a77797f36c683426abb41e
+  metadata.gz: bfdadf90a17ca05fbad43a5d38efc22179415f06b661b587cf5b087d0c200174
+  data.tar.gz: ca814528eaef23167aca0003fa62e180944ce3bc686d54728c9c30f82732a385
 SHA512:
-  metadata.gz: 80108514d02ec16533f76b172df6f5d350dcd1b3d4253bda633ebbc4fc84398e3d5073e8d47f155a2b9f42da69b008b1650f48c612759314807bc9e9d0488c2c
-  data.tar.gz: de622970e9b943f26933934de1784d99c0f788607f7b9ae8d183601a55dba61b514d0d9b0813b09590de35bb74a985d62a5b891834a18f4319e6b740e62c9517
+  metadata.gz: 680c478f59a3feb96c2ed27c6d169c3ad78d37191acf374dff456365a12611b3dba4c0b077e29808d55867446ab2a5008eca6db11f0707aa1a38caea5561d8ee
+  data.tar.gz: d0e80fc907715d307d6657f6f507643f732c417dc59f888c7123288a416f662a458aa15878516c79b4b9730ef5a306ba33d778354fc8d5cd7cbe448fd76a6a9c

data/.github/workflows/specs.yml

@@ -0,0 +1,62 @@
+name: Specs
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  all_specs:
+    name: All Specs
+    strategy:
+      matrix:
+        ruby: ['2.4.4', '2.5.1', '2.6.3']
+        gemfile: ['Gemfile', 'Gemfile.aws-sdk-core-v2']
+    runs-on: ubuntu-20.04
+    services:
+      moto_sqs:
+        image: quay.io/cjlarose/moto-sqs-server:1.1.0
+        ports:
+          - 5000:5000
+    env:
+      BUNDLE_GEMFILE: ${{ matrix.gemfile }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Run specs
+        run: bundle exec rake spec
+      - name: Run integration specs
+        run: bundle exec rake spec:integration
+  rails_specs:
+    name: Rails Specs
+    strategy:
+      matrix:
+        rails: ['4.2', '5.2', '6.0', '6.1']
+        include:
+          - rails: '4.2'
+            ruby: '2.2'
+            gemfile: gemfiles/rails_4_2.gemfile
+          - rails: '5.2'
+            ruby: '2.5'
+            gemfile: gemfiles/rails_5_2.gemfile
+          - rails: '6.0'
+            ruby: '2.6'
+            gemfile: gemfiles/rails_6_0.gemfile
+          - rails: '6.1'
+            ruby: '3.0'
+            gemfile: gemfiles/rails_6_1.gemfile
+    runs-on: ubuntu-20.04
+    env:
+      BUNDLE_GEMFILE: ${{ matrix.gemfile }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Run Rails specs
+        run: bundle exec rake spec:rails

data/.reek.yml

@@ -0,0 +1,5 @@
+---
+detectors:
+
+  UtilityFunction:
+    public_methods_only: true

data/.rubocop.yml

@@ -1,7 +1,7 @@
 AllCops:
   Exclude:
     - '**/Gemfile'
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.4
 
 Metrics/PerceivedComplexity:
   Enabled: false

data/Appraisals

@@ -0,0 +1,28 @@
+appraise 'rails_4_2' do
+  group :test do
+    gem 'activejob', '~> 4.2'
+  end
+
+  group :development do
+    gem 'appraisal', '~> 2.2'
+    remove_gem 'pry-byebug'
+  end
+end
+
+appraise 'rails_5_2' do
+  group :test do
+    gem 'activejob', '~> 5.2'
+  end
+end
+
+appraise 'rails_6_0' do
+  group :test do
+    gem 'activejob', '~> 6.0'
+  end
+end
+
+appraise 'rails_6_1' do
+  group :test do
+    gem 'activejob', '~> 6.1'
+  end
+end

data/CHANGELOG.md

@@ -1,3 +1,56 @@
+## [v5.2.1] - 2021-04-06
+
+- Reduce message batch sizes in `shoryuken sqs requeue` and `shoryuken sqs mv` commands
+  - [#666](https://github.com/ruby-shoryuken/shoryuken/pull/666)
+
+- Fix bug in `shoryuken sqs requeue` and `shoryuken sqs mv` where those commands would exceed the SQS `SendMessageBatch` maximum payload size
+  - [#663](https://github.com/ruby-shoryuken/shoryuken/issues/663)
+  - [#664](https://github.com/ruby-shoryuken/shoryuken/pull/664)
+
+- Remove test stub for `Concurrent.global_io_executor`
+  - [#662](https://github.com/ruby-shoryuken/shoryuken/pull/662)
+
+- Run integration tests on CI
+  - [#660](https://github.com/ruby-shoryuken/shoryuken/pull/660)
+
+## [v5.2.0] - 2021-02-26
+
+- Set `executions` correctly for ActiveJob jobs
+  - [#657](https://github.com/phstc/shoryuken/pull/657)
+
+## [v5.1.1] - 2021-02-10
+
+- Fix regression in Ruby 3.0 introduced in Shoryuken 5.1.0, where enqueueing jobs with ActiveJob to workers that used keyword arguments would fail
+  - [#654](https://github.com/phstc/shoryuken/pull/654)
+
+## [v5.1.0] - 2021-02-06
+
+- Add support for specifying SQS SendMessage parameters with ActiveJob `.set`
+  - [#635](https://github.com/phstc/shoryuken/pull/635)
+  - [#648](https://github.com/phstc/shoryuken/pull/648)
+  - [#651](https://github.com/phstc/shoryuken/pull/651)
+
+- Unpause FIFO queues on worker completion
+  - [#644](https://github.com/phstc/shoryuken/pull/644)
+
+- Add multiple versions of Rails to test matrix
+  - [#647](https://github.com/phstc/shoryuken/pull/647)
+
+- Migrate from Travis CI to Github Actions
+  - [#649](https://github.com/phstc/shoryuken/pull/649)
+  - [#650](https://github.com/phstc/shoryuken/pull/650)
+  - [#652](https://github.com/phstc/shoryuken/pull/652)
+
+## [v5.0.6] - 2020-12-30
+
+- Load ShoryukenConcurrentSendAdapter when loading Rails
+  - [#642](https://github.com/phstc/shoryuken/pull/642)
+
+## [v5.0.5] - 2020-06-07
+
+- Add ability to configure queue by ARN
+  - [#603](https://github.com/phstc/shoryuken/pull/603)
+
 ## [v5.0.4] - 2020-02-20
 
 - Add endpoint option to SQS CLI

data/Gemfile

@@ -4,7 +4,7 @@ source 'https://rubygems.org'
 gemspec
 
 group :test do
-  gem 'activejob', '~> 4'
+  gem 'activejob'
   gem 'aws-sdk-core', '~> 3'
   gem 'aws-sdk-sqs'
   gem 'codeclimate-test-reporter', require: nil
@@ -14,5 +14,7 @@ group :test do
 end
 
 group :development do
+  gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
+  gem 'pry-byebug', '3.9.0'
   gem 'rubocop'
 end

data/README.md

@@ -1,10 +1,12 @@
+**I'm looking for Shoryuken maintainers, are you interested on helping to maintain Shoryuken? Fill up this form https://forms.gle/8kTso8ixa9Sfp6rJ9**
+
 # Shoryuken
 
 ![Shoryuken](shoryuken.jpg)
 
 Shoryuken _sho-ryu-ken_ is a super-efficient [Amazon SQS](https://aws.amazon.com/sqs/) thread-based message processor.
 
-[![Build Status](https://travis-ci.org/phstc/shoryuken.svg)](https://travis-ci.org/phstc/shoryuken)
+[![Build Status](https://github.com/ruby-shoryuken/shoryuken/workflows/Specs/badge.svg)](https://github.com/ruby-shoryuken/shoryuken/actions)
 [![Code Climate](https://codeclimate.com/github/phstc/shoryuken/badges/gpa.svg)](https://codeclimate.com/github/phstc/shoryuken)
 
 ## Key features
@@ -21,7 +23,7 @@ Shoryuken _sho-ryu-ken_ is a super-efficient [Amazon SQS](https://aws.amazon.com
 
 ## Requirements
 
-Ruby 2.0 or greater.
+Ruby 2.4 or greater.
 
 ## Installation
 
@@ -64,3 +66,23 @@ For more information check the [wiki page](https://github.com/phstc/shoryuken/wi
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create a new Pull Request
+
+### Testing
+
+To run all unit specs against the latest dependency vesions, execute
+
+```sh
+bundle exec rake spec
+```
+
+To run all Rails-related specs against all supported versions of Rails, execute
+
+```sh
+bundle exec appraisal rake spec:rails
+```
+
+To run integration specs, start a mock SQS server on `localhost:5000`. One such option is [cjlarose/moto-sqs-server](https://github.com/cjlarose/moto-sqs-server). Then execute
+
+```sh
+bundle exec rake spec:integration
+```

data/Rakefile

@@ -3,7 +3,21 @@ $stdout.sync = true
 
 begin
   require 'rspec/core/rake_task'
-  RSpec::Core::RakeTask.new(:spec)
+  RSpec::Core::RakeTask.new(:spec) do |t|
+    t.exclude_pattern = 'spec/integration/**/*_spec.rb'
+  end
+
+  namespace :spec do
+    desc 'Run Rails specs only'
+    RSpec::Core::RakeTask.new(:rails) do |t|
+      t.pattern = 'spec/shoryuken/{environment_loader_spec,extensions/active_job_*}.rb'
+    end
+
+    desc 'Run integration specs only'
+    RSpec::Core::RakeTask.new(:integration) do |t|
+      t.pattern = 'spec/integration/**/*_spec.rb'
+    end
+  end
 rescue LoadError
 end
 

data/bin/cli/sqs.rb

@@ -4,6 +4,9 @@ require 'date'
 module Shoryuken
   module CLI
     class SQS < Base
+      # See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
+      MAX_BATCH_SIZE = 256 * 1024
+
       namespace :sqs
       class_option :endpoint, aliases: '-e', type: :string, default: ENV['SHORYUKEN_SQS_ENDPOINT'], desc: 'Endpoint URL'
 
@@ -51,14 +54,56 @@ module Shoryuken
           end
         end
 
-        def batch_send(url, messages, messages_per_batch = 10)
-          messages.to_a.flatten.map(&method(:normalize_dump_message)).each_slice(messages_per_batch) do |batch|
-            sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
-              say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
+        def batch_send(url, messages, max_batch_size = 10)
+          messages = messages.to_a.flatten.map(&method(:normalize_dump_message))
+          batch_send_normalized_messages url, messages, max_batch_size
+        end
+
+        def batch_send_normalized_messages(url, messages, max_batch_size)
+          # Repeatedly take the longest prefix of messages such that
+          # 1. The number of messages is less than or equal to max_batch_size
+          # 2. The total message payload size is less than or equal to the
+          #    batch payload limit
+          while messages.size.positive?
+            batch_size = max_batch_size
+            loop do
+              batch = messages.take batch_size
+
+              unless batch.size == 1 || batch_payload_size(batch) <= MAX_BATCH_SIZE
+                batch_size = batch.size - 1
+                next
+              end
+
+              sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
+                say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
+              end
+              messages = messages.drop batch.size
+              break
             end
           end
         end
 
+        def batch_payload_size(messages)
+          messages.sum(&method(:message_size))
+        end
+
+        def message_size(message)
+          attribute_size = (message[:message_attributes] || []).sum do |name, value|
+            name_size = name.to_s.bytesize
+            data_type_size = value[:data_type].bytesize
+            value_size = if value[:string_value]
+                           value[:string_value].bytesize
+                         elsif value[:binary_value]
+                           value[:binary_value].bytesize
+                         end
+            name_size + data_type_size + value_size
+          end
+
+          body_size = message[:message_body].bytesize
+
+          attribute_size + body_size
+        end
+
         def find_all(url, limit)
           count = 0
           batch_size = limit > 10 ? 10 : limit
@@ -160,7 +205,7 @@ module Shoryuken
       end
 
       desc 'requeue QUEUE-NAME PATH', 'Requeues messages from a dump file'
-      method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'number of messages per batch to send'
+      method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'maximum number of messages per batch to send'
       def requeue(queue_name, path)
         fail_task "Path #{path} not found" unless File.exist?(path)
 

data/gemfiles/.gitignore

@@ -0,0 +1 @@
+*.gemfile.lock

data/gemfiles/rails_4_2.gemfile

@@ -0,0 +1,20 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 4.2"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", "~> 2.2"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/gemfiles/rails_5_2.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 5.2"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/gemfiles/rails_6_0.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 6.0"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/gemfiles/rails_6_1.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 6.1"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/lib/shoryuken.rb

@@ -89,6 +89,7 @@ module Shoryuken
 end
 
 if Shoryuken.active_job?
+  require 'shoryuken/extensions/active_job_extensions'
   require 'shoryuken/extensions/active_job_adapter'
   require 'shoryuken/extensions/active_job_concurrent_send_adapter'
 end