shoryuken 5.2.0 → 5.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c855ac4349659f917c53b48687c306ded163e5e0bcab0437eddca57544f4eb0f
-  data.tar.gz: 2e7494359335eee6f17f663fb43922ff4d5ef1944256a88472efe9a0b7391347
+  metadata.gz: bfdadf90a17ca05fbad43a5d38efc22179415f06b661b587cf5b087d0c200174
+  data.tar.gz: ca814528eaef23167aca0003fa62e180944ce3bc686d54728c9c30f82732a385
 SHA512:
-  metadata.gz: f6ab1140e26468002b6f1ca41a57296d8168fbf19caf4d5f47ec3e51e89c48b1a961a271d0a416b3f6f1b81b27a7fea775bf5a950fa9658fdcf6b97c095f571e
-  data.tar.gz: 4ea7336a630cc887e82b7adcbafda5ed373c7fa5b82147b6f7c0038099980c6c86457df6471507f15189ad2391cccd6224a898f91c086e6bdcb9053b858d9114
+  metadata.gz: 680c478f59a3feb96c2ed27c6d169c3ad78d37191acf374dff456365a12611b3dba4c0b077e29808d55867446ab2a5008eca6db11f0707aa1a38caea5561d8ee
+  data.tar.gz: d0e80fc907715d307d6657f6f507643f732c417dc59f888c7123288a416f662a458aa15878516c79b4b9730ef5a306ba33d778354fc8d5cd7cbe448fd76a6a9c

data/.github/workflows/specs.yml

@@ -12,6 +12,11 @@ jobs:
         ruby: ['2.4.4', '2.5.1', '2.6.3']
         gemfile: ['Gemfile', 'Gemfile.aws-sdk-core-v2']
     runs-on: ubuntu-20.04
+    services:
+      moto_sqs:
+        image: quay.io/cjlarose/moto-sqs-server:1.1.0
+        ports:
+          - 5000:5000
     env:
       BUNDLE_GEMFILE: ${{ matrix.gemfile }}
     steps:
@@ -22,9 +27,9 @@ jobs:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
       - name: Run specs
-        env:
-          SPEC_ALL: true
         run: bundle exec rake spec
+      - name: Run integration specs
+        run: bundle exec rake spec:integration
   rails_specs:
     name: Rails Specs
     strategy:
@@ -54,4 +59,4 @@ jobs:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
       - name: Run Rails specs
-        run: bundle exec rake rails_specs
+        run: bundle exec rake spec:rails

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+## [v5.2.1] - 2021-04-06
+
+- Reduce message batch sizes in `shoryuken sqs requeue` and `shoryuken sqs mv` commands
+  - [#666](https://github.com/ruby-shoryuken/shoryuken/pull/666)
+
+- Fix bug in `shoryuken sqs requeue` and `shoryuken sqs mv` where those commands would exceed the SQS `SendMessageBatch` maximum payload size
+  - [#663](https://github.com/ruby-shoryuken/shoryuken/issues/663)
+  - [#664](https://github.com/ruby-shoryuken/shoryuken/pull/664)
+
+- Remove test stub for `Concurrent.global_io_executor`
+  - [#662](https://github.com/ruby-shoryuken/shoryuken/pull/662)
+
+- Run integration tests on CI
+  - [#660](https://github.com/ruby-shoryuken/shoryuken/pull/660)
+
 ## [v5.2.0] - 2021-02-26
 
 - Set `executions` correctly for ActiveJob jobs

data/README.md

@@ -69,7 +69,7 @@ For more information check the [wiki page](https://github.com/phstc/shoryuken/wi
 
 ### Testing
 
-To run all specs against the latest dependency vesions, execute
+To run all unit specs against the latest dependency vesions, execute
 
 ```sh
 bundle exec rake spec
@@ -78,5 +78,11 @@ bundle exec rake spec
 To run all Rails-related specs against all supported versions of Rails, execute
 
 ```sh
-bundle exec appraisal rake rails_specs
+bundle exec appraisal rake spec:rails
+```
+
+To run integration specs, start a mock SQS server on `localhost:5000`. One such option is [cjlarose/moto-sqs-server](https://github.com/cjlarose/moto-sqs-server). Then execute
+
+```sh
+bundle exec rake spec:integration
 ```

data/Rakefile

@@ -3,10 +3,21 @@ $stdout.sync = true
 
 begin
   require 'rspec/core/rake_task'
-  RSpec::Core::RakeTask.new(:spec)
+  RSpec::Core::RakeTask.new(:spec) do |t|
+    t.exclude_pattern = 'spec/integration/**/*_spec.rb'
+  end
+
+  namespace :spec do
+    desc 'Run Rails specs only'
+    RSpec::Core::RakeTask.new(:rails) do |t|
+      t.pattern = 'spec/shoryuken/{environment_loader_spec,extensions/active_job_*}.rb'
+    end
 
-  rails_task = RSpec::Core::RakeTask.new(:rails_specs)
-  rails_task.pattern = 'spec/shoryuken/{environment_loader_spec,extensions/active_job_*}.rb'
+    desc 'Run integration specs only'
+    RSpec::Core::RakeTask.new(:integration) do |t|
+      t.pattern = 'spec/integration/**/*_spec.rb'
+    end
+  end
 rescue LoadError
 end
 

data/bin/cli/sqs.rb

@@ -4,6 +4,9 @@ require 'date'
 module Shoryuken
   module CLI
     class SQS < Base
+      # See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
+      MAX_BATCH_SIZE = 256 * 1024
+
       namespace :sqs
       class_option :endpoint, aliases: '-e', type: :string, default: ENV['SHORYUKEN_SQS_ENDPOINT'], desc: 'Endpoint URL'
 
@@ -51,14 +54,56 @@ module Shoryuken
           end
         end
 
-        def batch_send(url, messages, messages_per_batch = 10)
-          messages.to_a.flatten.map(&method(:normalize_dump_message)).each_slice(messages_per_batch) do |batch|
-            sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
-              say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
+        def batch_send(url, messages, max_batch_size = 10)
+          messages = messages.to_a.flatten.map(&method(:normalize_dump_message))
+          batch_send_normalized_messages url, messages, max_batch_size
+        end
+
+        def batch_send_normalized_messages(url, messages, max_batch_size)
+          # Repeatedly take the longest prefix of messages such that
+          # 1. The number of messages is less than or equal to max_batch_size
+          # 2. The total message payload size is less than or equal to the
+          #    batch payload limit
+          while messages.size.positive?
+            batch_size = max_batch_size
+            loop do
+              batch = messages.take batch_size
+
+              unless batch.size == 1 || batch_payload_size(batch) <= MAX_BATCH_SIZE
+                batch_size = batch.size - 1
+                next
+              end
+
+              sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
+                say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
+              end
+              messages = messages.drop batch.size
+              break
             end
           end
         end
 
+        def batch_payload_size(messages)
+          messages.sum(&method(:message_size))
+        end
+
+        def message_size(message)
+          attribute_size = (message[:message_attributes] || []).sum do |name, value|
+            name_size = name.to_s.bytesize
+            data_type_size = value[:data_type].bytesize
+            value_size = if value[:string_value]
+                           value[:string_value].bytesize
+                         elsif value[:binary_value]
+                           value[:binary_value].bytesize
+                         end
+            name_size + data_type_size + value_size
+          end
+
+          body_size = message[:message_body].bytesize
+
+          attribute_size + body_size
+        end
+
         def find_all(url, limit)
           count = 0
           batch_size = limit > 10 ? 10 : limit
@@ -160,7 +205,7 @@ module Shoryuken
       end
 
       desc 'requeue QUEUE-NAME PATH', 'Requeues messages from a dump file'
-      method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'number of messages per batch to send'
+      method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'maximum number of messages per batch to send'
       def requeue(queue_name, path)
         fail_task "Path #{path} not found" unless File.exist?(path)
 

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '5.2.0'.freeze
+  VERSION = '5.2.1'.freeze
 end

data/spec/integration/launcher_spec.rb

@@ -4,10 +4,37 @@ require 'shoryuken/launcher'
 require 'securerandom'
 
 RSpec.describe Shoryuken::Launcher do
-  describe 'Consuming messages', slow: true do
+  let(:sqs_client) do
+    Aws::SQS::Client.new(
+      region: 'us-east-1',
+      endpoint: 'http://localhost:5000',
+      access_key_id: 'fake',
+      secret_access_key: 'fake'
+    )
+  end
+
+  let(:executor) do
+    # We can't use Concurrent.global_io_executor in these tests since once you
+    # shut down a thread pool, you can't start it back up. Instead, we create
+    # one new thread pool executor for each spec. We use a new
+    # CachedThreadPool, since that most closely resembles
+    # Concurrent.global_io_executor
+    Concurrent::CachedThreadPool.new auto_terminate: true
+  end
+
+  describe 'Consuming messages' do
     before do
       Aws.config[:stub_responses] = false
-      Aws.config[:region] = 'us-east-1'
+
+      allow(Shoryuken).to receive(:launcher_executor).and_return(executor)
+
+      Shoryuken.configure_client do |config|
+        config.sqs_client = sqs_client
+      end
+
+      Shoryuken.configure_server do |config|
+        config.sqs_client = sqs_client
+      end
 
       StandardWorker.received_messages = 0
 

data/spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb

@@ -10,6 +10,10 @@ RSpec.describe ActiveJob::QueueAdapters::ShoryukenConcurrentSendAdapter do
   let(:error_handler) { -> {} }
   let(:success_handler) { -> {} }
 
+  before do
+    allow(Concurrent).to receive(:global_io_executor).and_return(Concurrent::ImmediateExecutor.new)
+  end
+
   subject { described_class.new(success_handler, error_handler) }
 
   context 'when success' do

data/spec/spec_helper.rb

@@ -32,9 +32,6 @@ class TestWorker
 end
 
 RSpec.configure do |config|
-  # TODO: Run these tests again on CI
-  config.filter_run_excluding slow: true
-
   config.before do
     Shoryuken::Client.class_variable_set :@@queues, {}
 
@@ -63,7 +60,6 @@ RSpec.configure do |config|
 
     Shoryuken.cache_visibility_timeout = false
 
-    allow(Concurrent).to receive(:global_io_executor).and_return(Concurrent::ImmediateExecutor.new)
     allow(Shoryuken).to receive(:active_job?).and_return(false)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 5.2.0
+  version: 5.2.1
 platform: ruby
 authors:
 - Pablo Cantero
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-26 00:00:00.000000000 Z
+date: 2021-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv