shoryuken 5.1.1 → 5.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d08b7287c9dcb2c488715f1313e11d9a9645c068d90fc160c76dbc57ff5d795
-  data.tar.gz: 1e6009538535c438a02267e948d5f3eacaedf4aa1283950733bfc51972dcf872
+  metadata.gz: 0f2f8c9e8573b699391a74a70d26756e106ccf72b3df9a6fccc32b72d13ec0cd
+  data.tar.gz: 6a009d5a5e571da339d52a08f4ccda19f144ce06132f4a65e95fabc39aeffad4
 SHA512:
-  metadata.gz: dfc89030844c19828865a6495a4dcf4ef8e46f361303c4c906a0f1d56b2e9fdc46baebcd9eb6ee826a65a173dde4ed2d1a2e1ebc8dae64a8ed131fc5ebe65f13
-  data.tar.gz: af2ac18ce6d1445fd261344220b72271d78e10c4034bf565c782d4baad44f946459348342b1fbcbaa75fbe8984ef2655aad78cad2a7128feee5603466fc736d5
+  metadata.gz: e359eee09d82a917c4e13541a121c61f78587421a7e5977052552a4f18512eeb9a2eb720ebc8adf323d0a33fd979b37e9c059a95b5b180eda93cab5a3e034f6b
+  data.tar.gz: 5217287d2b7f66125852ac0b824353d934a2a8568e2dc07ca3c98f8991cad1a0dafbda87967a407bc5efaea62b1cafa85708e1ea17ccb659b2743878d079a267

data/.github/workflows/specs.yml

@@ -9,9 +9,14 @@ jobs:
     name: All Specs
     strategy:
       matrix:
-        ruby: ['2.4.4', '2.5.1', '2.6.3']
-        gemfile: ['Gemfile', 'Gemfile.aws-sdk-core-v2']
+        ruby: ['2.4', '2.5', '2.6', '2.7', '3.0']
+        gemfile: ['Gemfile', 'gemfiles/aws_sdk_core_2.gemfile']
     runs-on: ubuntu-20.04
+    services:
+      moto_sqs:
+        image: quay.io/cjlarose/moto-sqs-server:1.1.0
+        ports:
+          - 5000:5000
     env:
       BUNDLE_GEMFILE: ${{ matrix.gemfile }}
     steps:
@@ -22,9 +27,9 @@ jobs:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
       - name: Run specs
-        env:
-          SPEC_ALL: true
         run: bundle exec rake spec
+      - name: Run integration specs
+        run: bundle exec rake spec:integration
   rails_specs:
     name: Rails Specs
     strategy:
@@ -54,4 +59,4 @@ jobs:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
       - name: Run Rails specs
-        run: bundle exec rake rails_specs
+        run: bundle exec rake spec:rails

data/Appraisals

@@ -1,3 +1,11 @@
+appraise 'aws_sdk_core_2' do
+  group :test do
+    gem 'aws-sdk-core', '~> 2'
+    gem 'webrick' # required for Ruby 3
+    remove_gem 'aws-sdk-sqs'
+  end
+end
+
 appraise 'rails_4_2' do
   group :test do
     gem 'activejob', '~> 4.2'

data/CHANGELOG.md

@@ -1,3 +1,33 @@
+## [v5.2.3] - 2021-07-29
+
+- Fire new `:utilization_update` event any time a worker pool's utilization changes
+  - [#673](https://github.com/ruby-shoryuken/shoryuken/pull/673)
+
+## [v5.2.2] - 2021-06-22
+
+- When using ActiveJob queue name prefixing, avoid applying prefix to queues configured with a URL or ARN
+  - [#667](https://github.com/ruby-shoryuken/shoryuken/pull/667)
+
+## [v5.2.1] - 2021-04-06
+
+- Reduce message batch sizes in `shoryuken sqs requeue` and `shoryuken sqs mv` commands
+  - [#666](https://github.com/ruby-shoryuken/shoryuken/pull/666)
+
+- Fix bug in `shoryuken sqs requeue` and `shoryuken sqs mv` where those commands would exceed the SQS `SendMessageBatch` maximum payload size
+  - [#663](https://github.com/ruby-shoryuken/shoryuken/issues/663)
+  - [#664](https://github.com/ruby-shoryuken/shoryuken/pull/664)
+
+- Remove test stub for `Concurrent.global_io_executor`
+  - [#662](https://github.com/ruby-shoryuken/shoryuken/pull/662)
+
+- Run integration tests on CI
+  - [#660](https://github.com/ruby-shoryuken/shoryuken/pull/660)
+
+## [v5.2.0] - 2021-02-26
+
+- Set `executions` correctly for ActiveJob jobs
+  - [#657](https://github.com/phstc/shoryuken/pull/657)
+
 ## [v5.1.1] - 2021-02-10
 
 - Fix regression in Ruby 3.0 introduced in Shoryuken 5.1.0, where enqueueing jobs with ActiveJob to workers that used keyword arguments would fail

data/README.md

@@ -69,7 +69,7 @@ For more information check the [wiki page](https://github.com/phstc/shoryuken/wi
 
 ### Testing
 
-To run all specs against the latest dependency vesions, execute
+To run all unit specs against the latest dependency vesions, execute
 
 ```sh
 bundle exec rake spec
@@ -78,5 +78,11 @@ bundle exec rake spec
 To run all Rails-related specs against all supported versions of Rails, execute
 
 ```sh
-bundle exec appraisal rake rails_specs
+bundle exec appraisal rake spec:rails
+```
+
+To run integration specs, start a mock SQS server on `localhost:5000`. One such option is [cjlarose/moto-sqs-server](https://github.com/cjlarose/moto-sqs-server). Then execute
+
+```sh
+bundle exec rake spec:integration
 ```

data/Rakefile

@@ -3,10 +3,21 @@ $stdout.sync = true
 
 begin
   require 'rspec/core/rake_task'
-  RSpec::Core::RakeTask.new(:spec)
+  RSpec::Core::RakeTask.new(:spec) do |t|
+    t.exclude_pattern = 'spec/integration/**/*_spec.rb'
+  end
+
+  namespace :spec do
+    desc 'Run Rails specs only'
+    RSpec::Core::RakeTask.new(:rails) do |t|
+      t.pattern = 'spec/shoryuken/{environment_loader_spec,extensions/active_job_*}.rb'
+    end
 
-  rails_task = RSpec::Core::RakeTask.new(:rails_specs)
-  rails_task.pattern = 'spec/shoryuken/{environment_loader_spec,extensions/active_job_*}.rb'
+    desc 'Run integration specs only'
+    RSpec::Core::RakeTask.new(:integration) do |t|
+      t.pattern = 'spec/integration/**/*_spec.rb'
+    end
+  end
 rescue LoadError
 end
 

data/bin/cli/sqs.rb

@@ -4,6 +4,9 @@ require 'date'
 module Shoryuken
   module CLI
     class SQS < Base
+      # See https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html
+      MAX_BATCH_SIZE = 256 * 1024
+
       namespace :sqs
       class_option :endpoint, aliases: '-e', type: :string, default: ENV['SHORYUKEN_SQS_ENDPOINT'], desc: 'Endpoint URL'
 
@@ -51,14 +54,56 @@ module Shoryuken
           end
         end
 
-        def batch_send(url, messages, messages_per_batch = 10)
-          messages.to_a.flatten.map(&method(:normalize_dump_message)).each_slice(messages_per_batch) do |batch|
-            sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
-              say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
+        def batch_send(url, messages, max_batch_size = 10)
+          messages = messages.to_a.flatten.map(&method(:normalize_dump_message))
+          batch_send_normalized_messages url, messages, max_batch_size
+        end
+
+        def batch_send_normalized_messages(url, messages, max_batch_size)
+          # Repeatedly take the longest prefix of messages such that
+          # 1. The number of messages is less than or equal to max_batch_size
+          # 2. The total message payload size is less than or equal to the
+          #    batch payload limit
+          while messages.size.positive?
+            batch_size = max_batch_size
+            loop do
+              batch = messages.take batch_size
+
+              unless batch.size == 1 || batch_payload_size(batch) <= MAX_BATCH_SIZE
+                batch_size = batch.size - 1
+                next
+              end
+
+              sqs.send_message_batch(queue_url: url, entries: batch).failed.any? do |failure|
+                say "Could not requeue #{failure.id}, code: #{failure.code}", :yellow
+              end
+              messages = messages.drop batch.size
+              break
             end
           end
         end
 
+        def batch_payload_size(messages)
+          messages.sum(&method(:message_size))
+        end
+
+        def message_size(message)
+          attribute_size = (message[:message_attributes] || []).sum do |name, value|
+            name_size = name.to_s.bytesize
+            data_type_size = value[:data_type].bytesize
+            value_size = if value[:string_value]
+                           value[:string_value].bytesize
+                         elsif value[:binary_value]
+                           value[:binary_value].bytesize
+                         end
+            name_size + data_type_size + value_size
+          end
+
+          body_size = message[:message_body].bytesize
+
+          attribute_size + body_size
+        end
+
         def find_all(url, limit)
           count = 0
           batch_size = limit > 10 ? 10 : limit
@@ -160,7 +205,7 @@ module Shoryuken
       end
 
       desc 'requeue QUEUE-NAME PATH', 'Requeues messages from a dump file'
-      method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'number of messages per batch to send'
+      method_option :batch_size, aliases: '-n', type: :numeric, default: 10, desc: 'maximum number of messages per batch to send'
       def requeue(queue_name, path)
         fail_task "Path #{path} not found" unless File.exist?(path)
 

data/gemfiles/aws_sdk_core_2.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob"
+  gem "aws-sdk-core", "~> 2"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+  gem "webrick"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/lib/shoryuken/environment_loader.rb

@@ -80,6 +80,8 @@ module Shoryuken
     end
 
     def prefix_active_job_queue_name(queue_name, weight)
+      return [queue_name, weight] if queue_name.start_with?('https://', 'arn:')
+
       queue_name_prefix = ::ActiveJob::Base.queue_name_prefix
       queue_name_delimiter = ::ActiveJob::Base.queue_name_delimiter
 

data/lib/shoryuken/extensions/active_job_adapter.rb

@@ -82,8 +82,10 @@ module ActiveJob
 
         shoryuken_options body_parser: :json, auto_delete: true
 
-        def perform(_sqs_msg, hash)
-          Base.execute hash
+        def perform(sqs_msg, hash)
+          receive_count = sqs_msg.attributes['ApproximateReceiveCount'].to_i
+          past_receives = receive_count - 1
+          Base.execute hash.merge({ 'executions' => past_receives })
         end
       end
 

data/lib/shoryuken/launcher.rb

@@ -71,6 +71,7 @@ module Shoryuken
     def create_managers
       Shoryuken.groups.map do |group, options|
         Shoryuken::Manager.new(
+          group,
           Shoryuken::Fetcher.new(group),
           Shoryuken.polling_strategy(group).new(options[:queues], Shoryuken.delay(group)),
           options[:concurrency],

data/lib/shoryuken/manager.rb

@@ -6,7 +6,8 @@ module Shoryuken
     # See https://github.com/phstc/shoryuken/issues/348#issuecomment-292847028
     MIN_DISPATCH_INTERVAL = 0.1
 
-    def initialize(fetcher, polling_strategy, concurrency, executor)
+    def initialize(group, fetcher, polling_strategy, concurrency, executor)
+      @group            = group
       @fetcher          = fetcher
       @polling_strategy = polling_strategy
       @max_processors   = concurrency
@@ -16,6 +17,7 @@ module Shoryuken
     end
 
     def start
+      fire_utilization_update_event
       dispatch_loop
     end
 
@@ -59,6 +61,8 @@ module Shoryuken
 
     def processor_done(queue)
       @busy_processors.decrement
+      fire_utilization_update_event
+
       client_queue = Shoryuken::Client.queues(queue)
       return unless client_queue.fifo?
       return unless @polling_strategy.respond_to?(:message_processed)
@@ -72,6 +76,7 @@ module Shoryuken
       logger.debug { "Assigning #{sqs_msg.message_id}" }
 
       @busy_processors.increment
+      fire_utilization_update_event
 
       Concurrent::Promise
         .execute(executor: @executor) { Processor.process(queue_name, sqs_msg) }
@@ -114,5 +119,13 @@ module Shoryuken
 
       @running.make_false
     end
+
+    def fire_utilization_update_event
+      fire_event :utilization_update, false, {
+        group: @group,
+        max_processors: @max_processors,
+        busy_processors: busy
+      }
+    end
   end
 end

data/lib/shoryuken/options.rb

@@ -9,6 +9,7 @@ module Shoryuken
       lifecycle_events: {
         startup: [],
         dispatch: [],
+        utilization_update: [],
         quiet: [],
         shutdown: []
       }

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '5.1.1'.freeze
+  VERSION = '5.2.3'.freeze
 end

data/spec/integration/launcher_spec.rb

@@ -4,10 +4,37 @@ require 'shoryuken/launcher'
 require 'securerandom'
 
 RSpec.describe Shoryuken::Launcher do
-  describe 'Consuming messages', slow: true do
+  let(:sqs_client) do
+    Aws::SQS::Client.new(
+      region: 'us-east-1',
+      endpoint: 'http://localhost:5000',
+      access_key_id: 'fake',
+      secret_access_key: 'fake'
+    )
+  end
+
+  let(:executor) do
+    # We can't use Concurrent.global_io_executor in these tests since once you
+    # shut down a thread pool, you can't start it back up. Instead, we create
+    # one new thread pool executor for each spec. We use a new
+    # CachedThreadPool, since that most closely resembles
+    # Concurrent.global_io_executor
+    Concurrent::CachedThreadPool.new auto_terminate: true
+  end
+
+  describe 'Consuming messages' do
     before do
       Aws.config[:stub_responses] = false
-      Aws.config[:region] = 'us-east-1'
+
+      allow(Shoryuken).to receive(:launcher_executor).and_return(executor)
+
+      Shoryuken.configure_client do |config|
+        config.sqs_client = sqs_client
+      end
+
+      Shoryuken.configure_server do |config|
+        config.sqs_client = sqs_client
+      end
 
       StandardWorker.received_messages = 0
 

data/spec/shoryuken/environment_loader_spec.rb

@@ -57,11 +57,11 @@ RSpec.describe Shoryuken::EnvironmentLoader do
       ActiveJob::Base.queue_name_delimiter = '_'
 
       allow(Shoryuken).to receive(:active_job?).and_return(true)
-    end
 
-    specify do
       Shoryuken.active_job_queue_name_prefixing = true
+    end
 
+    specify do
       Shoryuken.options[:queues] = ['queue1', ['queue2', 2]]
 
       Shoryuken.options[:groups] = {
@@ -73,6 +73,26 @@ RSpec.describe Shoryuken::EnvironmentLoader do
       expect(Shoryuken.groups['default'][:queues]).to eq(%w[test_queue1 test_queue2 test_queue2])
       expect(Shoryuken.groups['group1'][:queues]).to eq(%w[test_group1_queue1 test_group1_queue2])
     end
+
+    it 'does not prefix url-based queues' do
+      Shoryuken.options[:queues] = ['https://example.com/test_queue1']
+      Shoryuken.options[:groups] = {'group1' => {queues: ['https://example.com/test_group1_queue1']}}
+
+      subject.load
+
+      expect(Shoryuken.groups['default'][:queues]).to(eq(['https://example.com/test_queue1']))
+      expect(Shoryuken.groups['group1'][:queues]).to(eq(['https://example.com/test_group1_queue1']))
+    end
+
+    it 'does not prefix arn-based queues' do
+      Shoryuken.options[:queues] = ['arn:aws:sqs:fake-region-1:1234:test_queue1']
+      Shoryuken.options[:groups] = {'group1' => {queues: ['arn:aws:sqs:fake-region-1:1234:test_group1_queue1']}}
+
+      subject.load
+
+      expect(Shoryuken.groups['default'][:queues]).to(eq(['arn:aws:sqs:fake-region-1:1234:test_queue1']))
+      expect(Shoryuken.groups['group1'][:queues]).to(eq(['arn:aws:sqs:fake-region-1:1234:test_group1_queue1']))
+    end
   end
   describe "#setup_options" do
     let (:cli_queues) { { "queue1"=> 10, "queue2" => 20 } }

data/spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb

@@ -10,6 +10,10 @@ RSpec.describe ActiveJob::QueueAdapters::ShoryukenConcurrentSendAdapter do
   let(:error_handler) { -> {} }
   let(:success_handler) { -> {} }
 
+  before do
+    allow(Concurrent).to receive(:global_io_executor).and_return(Concurrent::ImmediateExecutor.new)
+  end
+
   subject { described_class.new(success_handler, error_handler) }
 
   context 'when success' do

data/spec/shoryuken/extensions/active_job_wrapper_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+require 'active_job'
+require 'shoryuken/extensions/active_job_extensions'
+require 'shoryuken/extensions/active_job_adapter'
+
+RSpec.describe ActiveJob::QueueAdapters::ShoryukenAdapter::JobWrapper do
+  subject { described_class.new }
+
+  describe '#perform' do
+    it 'sets executions to reflect approximate receive count' do
+      attributes = { 'ApproximateReceiveCount' => '42' }
+      sqs_msg = double Shoryuken::Message, attributes: attributes
+      job_hash = { 'arguments' => [1, 2, 3] }
+      job_hash_with_executions = { 'arguments' => [1, 2, 3], 'executions' => 41 }
+      expect(ActiveJob::Base).to receive(:execute).with(job_hash_with_executions)
+
+      subject.perform sqs_msg, job_hash
+    end
+  end
+end

data/spec/shoryuken/manager_spec.rb

@@ -15,7 +15,7 @@ RSpec.describe Shoryuken::Manager do
   let(:concurrency) { 1 }
   let(:executor) { Concurrent::ImmediateExecutor.new }
 
-  subject { Shoryuken::Manager.new(fetcher, polling_strategy, concurrency, executor) }
+  subject { Shoryuken::Manager.new('default', fetcher, polling_strategy, concurrency, executor) }
 
   before do
     allow(fetcher).to receive(:fetch).and_return([])
@@ -71,6 +71,11 @@ RSpec.describe Shoryuken::Manager do
 
       expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
       expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
+      expect(subject).to receive(:fire_event).with(:utilization_update,
+                                                   false,
+                                                   group: 'default',
+                                                   busy_processors: 1,
+                                                   max_processors: 1)
       expect(Shoryuken::Processor).to receive(:process).with(q, message)
       expect(Shoryuken.logger).to receive(:info).never
 
@@ -99,6 +104,11 @@ RSpec.describe Shoryuken::Manager do
         q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
 
         expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
+        expect(subject).to receive(:fire_event).with(:utilization_update,
+                                                     false,
+                                                     group: 'default',
+                                                     busy_processors: 1,
+                                                     max_processors: 1)
         expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
         allow(subject).to receive(:batched_queue?).with(q).and_return(true)
         expect(Shoryuken::Processor).to receive(:process).with(q, messages)

data/spec/spec_helper.rb

@@ -32,9 +32,6 @@ class TestWorker
 end
 
 RSpec.configure do |config|
-  # TODO: Run these tests again on CI
-  config.filter_run_excluding slow: true
-
   config.before do
     Shoryuken::Client.class_variable_set :@@queues, {}
 
@@ -63,7 +60,6 @@ RSpec.configure do |config|
 
     Shoryuken.cache_visibility_timeout = false
 
-    allow(Concurrent).to receive(:global_io_executor).and_return(Concurrent::ImmediateExecutor.new)
     allow(Shoryuken).to receive(:active_job?).and_return(false)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 5.1.1
+  version: 5.2.3
 platform: ruby
 authors:
 - Pablo Cantero
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-11 00:00:00.000000000 Z
+date: 2021-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv
@@ -112,7 +112,6 @@ files:
 - Appraisals
 - CHANGELOG.md
 - Gemfile
-- Gemfile.aws-sdk-core-v2
 - LICENSE
 - README.md
 - Rakefile
@@ -122,6 +121,7 @@ files:
 - examples/bootstrap_queues.rb
 - examples/default_worker.rb
 - gemfiles/.gitignore
+- gemfiles/aws_sdk_core_2.gemfile
 - gemfiles/rails_4_2.gemfile
 - gemfiles/rails_5_2.gemfile
 - gemfiles/rails_6_0.gemfile
@@ -172,6 +172,7 @@ files:
 - spec/shoryuken/extensions/active_job_adapter_spec.rb
 - spec/shoryuken/extensions/active_job_base_spec.rb
 - spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb
+- spec/shoryuken/extensions/active_job_wrapper_spec.rb
 - spec/shoryuken/fetcher_spec.rb
 - spec/shoryuken/manager_spec.rb
 - spec/shoryuken/middleware/chain_spec.rb
@@ -228,6 +229,7 @@ test_files:
 - spec/shoryuken/extensions/active_job_adapter_spec.rb
 - spec/shoryuken/extensions/active_job_base_spec.rb
 - spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb
+- spec/shoryuken/extensions/active_job_wrapper_spec.rb
 - spec/shoryuken/fetcher_spec.rb
 - spec/shoryuken/manager_spec.rb
 - spec/shoryuken/middleware/chain_spec.rb

data/Gemfile.aws-sdk-core-v2

@@ -1,13 +0,0 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in shoryuken.gemspec
-gemspec
-
-group :test do
-  gem 'activejob', '~> 4'
-  gem 'aws-sdk-core', '~> 2'
-  gem 'codeclimate-test-reporter', require: nil
-  gem 'httparty'
-  gem 'multi_xml'
-  gem 'simplecov'
-end