shoryuken 5.2.1 → 5.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bfdadf90a17ca05fbad43a5d38efc22179415f06b661b587cf5b087d0c200174
-  data.tar.gz: ca814528eaef23167aca0003fa62e180944ce3bc686d54728c9c30f82732a385
+  metadata.gz: c730786f3c97f263b00320693e0ea95af12495bb83805a39461de93b526b4d2a
+  data.tar.gz: a7a7fbb4bd5a1c006c1573261c607c6c77968575af901a8671dea4da6e4282c5
 SHA512:
-  metadata.gz: 680c478f59a3feb96c2ed27c6d169c3ad78d37191acf374dff456365a12611b3dba4c0b077e29808d55867446ab2a5008eca6db11f0707aa1a38caea5561d8ee
-  data.tar.gz: d0e80fc907715d307d6657f6f507643f732c417dc59f888c7123288a416f662a458aa15878516c79b4b9730ef5a306ba33d778354fc8d5cd7cbe448fd76a6a9c
+  metadata.gz: 3bc9237605e5f9667f0e22bdae6ca1553b2f2a08357f2f7da465fa4f812d3abcafd6ef21c05ef553dac13dcb405fa658d13bada5be235743974978fc7829b6d2
+  data.tar.gz: 54737eb8fc32b4a486d62f1ef626336e0493ae70475efb1a99e163ebd960d2e38c99506ccf8cafcd3502bd52a2279619601150634e7dbf3134b38913bb0283e8

data/.github/workflows/specs.yml

@@ -9,8 +9,8 @@ jobs:
     name: All Specs
     strategy:
       matrix:
-        ruby: ['2.4.4', '2.5.1', '2.6.3']
-        gemfile: ['Gemfile', 'Gemfile.aws-sdk-core-v2']
+        ruby: ['2.4', '2.5', '2.6', '2.7', '3.0']
+        gemfile: ['Gemfile', 'gemfiles/aws_sdk_core_2.gemfile']
     runs-on: ubuntu-20.04
     services:
       moto_sqs:

data/.gitignore

@@ -25,4 +25,4 @@ shoryuken.yml
 *.log
 .env
 rubocop.html
-.byebug_history
+.byebug_history

data/.rubocop.yml

@@ -23,7 +23,7 @@ Metrics/AbcSize:
   # because codeclimate already give that for us with more details
   Enabled: false
 
-Metrics/LineLength:
+Layout/LineLength:
   Max: 125
 
 Style/Alias:

data/Appraisals

@@ -1,3 +1,11 @@
+appraise 'aws_sdk_core_2' do
+  group :test do
+    gem 'aws-sdk-core', '~> 2'
+    gem 'webrick' # required for Ruby 3
+    remove_gem 'aws-sdk-sqs'
+  end
+end
+
 appraise 'rails_4_2' do
   group :test do
     gem 'activejob', '~> 4.2'

data/CHANGELOG.md

@@ -1,3 +1,26 @@
+## [v5.3.1] - 2022-01-07
+
+- (Bugfix) Fix issue where, when using the TSTP or USR1 signals for soft shutdowns, it was possible for shoryuken to terminate without first attempting to handle all messages it fetched from SQS
+  - [#676](https://github.com/ruby-shoryuken/shoryuken/pull/676)
+
+## [v5.3.0] - 2021-10-31
+
+- (Refactor) Use Forwardable within Message to avoid method boilerplate
+  - [#681](https://github.com/ruby-shoryuken/shoryuken/pull/681)
+
+- Add basic health check API
+  - [#679](https://github.com/ruby-shoryuken/shoryuken/pull/679)
+
+## [v5.2.3] - 2021-07-29
+
+- Fire new `:utilization_update` event any time a worker pool's utilization changes
+  - [#673](https://github.com/ruby-shoryuken/shoryuken/pull/673)
+
+## [v5.2.2] - 2021-06-22
+
+- When using ActiveJob queue name prefixing, avoid applying prefix to queues configured with a URL or ARN
+  - [#667](https://github.com/ruby-shoryuken/shoryuken/pull/667)
+
 ## [v5.2.1] - 2021-04-06
 
 - Reduce message batch sizes in `shoryuken sqs requeue` and `shoryuken sqs mv` commands

data/Gemfile

@@ -16,5 +16,5 @@ end
 group :development do
   gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
   gem 'pry-byebug', '3.9.0'
-  gem 'rubocop'
+  gem 'rubocop', '<= 1.12'
 end

data/gemfiles/aws_sdk_core_2.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob"
+  gem "aws-sdk-core", "~> 2"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+  gem "webrick"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/lib/shoryuken/environment_loader.rb

@@ -80,6 +80,8 @@ module Shoryuken
     end
 
     def prefix_active_job_queue_name(queue_name, weight)
+      return [queue_name, weight] if queue_name.start_with?('https://', 'arn:')
+
       queue_name_prefix = ::ActiveJob::Base.queue_name_prefix
       queue_name_delimiter = ::ActiveJob::Base.queue_name_delimiter
 

data/lib/shoryuken/launcher.rb

@@ -28,12 +28,30 @@ module Shoryuken
 
       initiate_stop
 
+      stop_new_dispatching
+      await_dispatching_in_progress
+
       executor.shutdown
       executor.wait_for_termination
     end
 
+    def healthy?
+      Shoryuken.groups.keys.all? do |group|
+        manager = @managers.find { |m| m.group == group }
+        manager && manager.running?
+      end
+    end
+
     private
 
+    def stop_new_dispatching
+      @managers.each(&:stop_new_dispatching)
+    end
+
+    def await_dispatching_in_progress
+      @managers.each(&:await_dispatching_in_progress)
+    end
+
     def executor
       @_executor ||= Shoryuken.launcher_executor || Concurrent.global_io_executor
     end
@@ -71,6 +89,7 @@ module Shoryuken
     def create_managers
       Shoryuken.groups.map do |group, options|
         Shoryuken::Manager.new(
+          group,
           Shoryuken::Fetcher.new(group),
           Shoryuken.polling_strategy(group).new(options[:queues], Shoryuken.delay(group)),
           options[:concurrency],

data/lib/shoryuken/manager.rb

@@ -6,27 +6,47 @@ module Shoryuken
     # See https://github.com/phstc/shoryuken/issues/348#issuecomment-292847028
     MIN_DISPATCH_INTERVAL = 0.1
 
-    def initialize(fetcher, polling_strategy, concurrency, executor)
-      @fetcher          = fetcher
-      @polling_strategy = polling_strategy
-      @max_processors   = concurrency
-      @busy_processors  = Concurrent::AtomicFixnum.new(0)
-      @executor         = executor
-      @running          = Concurrent::AtomicBoolean.new(true)
+    attr_reader :group
+
+    def initialize(group, fetcher, polling_strategy, concurrency, executor)
+      @group                      = group
+      @fetcher                    = fetcher
+      @polling_strategy           = polling_strategy
+      @max_processors             = concurrency
+      @busy_processors            = Concurrent::AtomicFixnum.new(0)
+      @executor                   = executor
+      @running                    = Concurrent::AtomicBoolean.new(true)
+      @stop_new_dispatching       = Concurrent::AtomicBoolean.new(false)
+      @dispatching_release_signal = ::Queue.new
     end
 
     def start
+      fire_utilization_update_event
       dispatch_loop
     end
 
-    private
+    def stop_new_dispatching
+      @stop_new_dispatching.make_true
+    end
+
+    def await_dispatching_in_progress
+      # There might still be a dispatching on-going, as the response from SQS could take some time
+      # We don't want to stop the process before processing incoming messages, as they would stay "in-flight" for some time on SQS
+      # We use a queue, as the dispatch_loop is running on another thread, and this is a efficient way of communicating between threads.
+      @dispatching_release_signal.pop
+    end
 
     def running?
       @running.true? && @executor.running?
     end
 
+    private
+
     def dispatch_loop
-      return unless running?
+      if @stop_new_dispatching.true? || !running?
+        @dispatching_release_signal << 1
+        return
+      end
 
       @executor.post { dispatch }
     end
@@ -59,6 +79,8 @@ module Shoryuken
 
     def processor_done(queue)
       @busy_processors.decrement
+      fire_utilization_update_event
+
       client_queue = Shoryuken::Client.queues(queue)
       return unless client_queue.fifo?
       return unless @polling_strategy.respond_to?(:message_processed)
@@ -72,6 +94,7 @@ module Shoryuken
       logger.debug { "Assigning #{sqs_msg.message_id}" }
 
       @busy_processors.increment
+      fire_utilization_update_event
 
       Concurrent::Promise
         .execute(executor: @executor) { Processor.process(queue_name, sqs_msg) }
@@ -87,7 +110,6 @@ module Shoryuken
 
     def dispatch_single_messages(queue)
       messages = @fetcher.fetch(queue, ready)
-
       @polling_strategy.messages_found(queue.name, messages.size)
       messages.each { |message| assign(queue.name, message) }
     end
@@ -114,5 +136,13 @@ module Shoryuken
 
       @running.make_false
     end
+
+    def fire_utilization_update_event
+      fire_event :utilization_update, false, {
+        group: @group,
+        max_processors: @max_processors,
+        busy_processors: busy
+      }
+    end
   end
 end

data/lib/shoryuken/message.rb

@@ -1,5 +1,16 @@
 module Shoryuken
   class Message
+    extend Forwardable
+
+    def_delegators(:data,
+                   :message_id,
+                   :receipt_handle,
+                   :md5_of_body,
+                   :body,
+                   :attributes,
+                   :md5_of_message_attributes,
+                   :message_attributes)
+
     attr_accessor :client, :queue_url, :queue_name, :data
 
     def initialize(client, queue, data)
@@ -29,33 +40,5 @@ module Shoryuken
         visibility_timeout: timeout
       )
     end
-
-    def message_id
-      data.message_id
-    end
-
-    def receipt_handle
-      data.receipt_handle
-    end
-
-    def md5_of_body
-      data.md5_of_body
-    end
-
-    def body
-      data.body
-    end
-
-    def attributes
-      data.attributes
-    end
-
-    def md5_of_message_attributes
-      data.md5_of_message_attributes
-    end
-
-    def message_attributes
-      data.message_attributes
-    end
   end
 end

data/lib/shoryuken/options.rb

@@ -9,6 +9,7 @@ module Shoryuken
       lifecycle_events: {
         startup: [],
         dispatch: [],
+        utilization_update: [],
         quiet: [],
         shutdown: []
       }

data/lib/shoryuken/runner.rb

@@ -55,6 +55,10 @@ module Shoryuken
       end
     end
 
+    def healthy?
+      (@launcher && @launcher.healthy?) || false
+    end
+
     private
 
     def initialize_concurrent_logger

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '5.2.1'.freeze
+  VERSION = '5.3.1'.freeze
 end

data/lib/shoryuken.rb

@@ -45,6 +45,10 @@ module Shoryuken
     @_shoryuken_options ||= Shoryuken::Options.new
   end
 
+  def self.healthy?
+    Shoryuken::Runner.instance.healthy?
+  end
+
   def_delegators(
     :shoryuken_options,
     :active_job?,

data/spec/shoryuken/environment_loader_spec.rb

@@ -57,11 +57,11 @@ RSpec.describe Shoryuken::EnvironmentLoader do
       ActiveJob::Base.queue_name_delimiter = '_'
 
       allow(Shoryuken).to receive(:active_job?).and_return(true)
-    end
 
-    specify do
       Shoryuken.active_job_queue_name_prefixing = true
+    end
 
+    specify do
       Shoryuken.options[:queues] = ['queue1', ['queue2', 2]]
 
       Shoryuken.options[:groups] = {
@@ -73,6 +73,26 @@ RSpec.describe Shoryuken::EnvironmentLoader do
       expect(Shoryuken.groups['default'][:queues]).to eq(%w[test_queue1 test_queue2 test_queue2])
       expect(Shoryuken.groups['group1'][:queues]).to eq(%w[test_group1_queue1 test_group1_queue2])
     end
+
+    it 'does not prefix url-based queues' do
+      Shoryuken.options[:queues] = ['https://example.com/test_queue1']
+      Shoryuken.options[:groups] = {'group1' => {queues: ['https://example.com/test_group1_queue1']}}
+
+      subject.load
+
+      expect(Shoryuken.groups['default'][:queues]).to(eq(['https://example.com/test_queue1']))
+      expect(Shoryuken.groups['group1'][:queues]).to(eq(['https://example.com/test_group1_queue1']))
+    end
+
+    it 'does not prefix arn-based queues' do
+      Shoryuken.options[:queues] = ['arn:aws:sqs:fake-region-1:1234:test_queue1']
+      Shoryuken.options[:groups] = {'group1' => {queues: ['arn:aws:sqs:fake-region-1:1234:test_group1_queue1']}}
+
+      subject.load
+
+      expect(Shoryuken.groups['default'][:queues]).to(eq(['arn:aws:sqs:fake-region-1:1234:test_queue1']))
+      expect(Shoryuken.groups['group1'][:queues]).to(eq(['arn:aws:sqs:fake-region-1:1234:test_group1_queue1']))
+    end
   end
   describe "#setup_options" do
     let (:cli_queues) { { "queue1"=> 10, "queue2" => 20 } }

data/spec/shoryuken/launcher_spec.rb

@@ -0,0 +1,60 @@
+require 'spec_helper'
+require 'shoryuken/launcher'
+
+RSpec.describe Shoryuken::Launcher do
+  let(:executor) do
+    # We can't use Concurrent.global_io_executor in these tests since once you
+    # shut down a thread pool, you can't start it back up. Instead, we create
+    # one new thread pool executor for each spec. We use a new
+    # CachedThreadPool, since that most closely resembles
+    # Concurrent.global_io_executor
+    Concurrent::CachedThreadPool.new auto_terminate: true
+  end
+
+  let(:first_group_manager) { double(:first_group_manager, group: 'first_group') }
+  let(:second_group_manager) { double(:second_group_manager, group: 'second_group') }
+  let(:first_queue) { "launcher_spec_#{SecureRandom.uuid}" }
+  let(:second_queue) { "launcher_spec_#{SecureRandom.uuid}" }
+
+  before do
+    Shoryuken.add_group('first_group', 1)
+    Shoryuken.add_group('second_group', 1)
+    Shoryuken.add_queue(first_queue, 1, 'first_group')
+    Shoryuken.add_queue(second_queue, 1, 'second_group')
+    allow(Shoryuken).to receive(:launcher_executor).and_return(executor)
+    allow(Shoryuken::Manager).to receive(:new).with('first_group', any_args).and_return(first_group_manager)
+    allow(Shoryuken::Manager).to receive(:new).with('second_group', any_args).and_return(second_group_manager)
+    allow(first_group_manager).to receive(:running?).and_return(true)
+    allow(second_group_manager).to receive(:running?).and_return(true)
+  end
+
+  describe '#healthy?' do
+    context 'when all groups have managers' do
+      context 'when all managers are running' do
+        it 'returns true' do
+          expect(subject.healthy?).to be true
+        end
+      end
+
+      context 'when one manager is not running' do
+        before do
+          allow(second_group_manager).to receive(:running?).and_return(false)
+        end
+
+        it 'returns false' do
+          expect(subject.healthy?).to be false
+        end
+      end
+    end
+
+    context 'when all groups do not have managers' do
+      before do
+        allow(second_group_manager).to receive(:group).and_return('some_random_group')
+      end
+
+      it 'returns false' do
+        expect(subject.healthy?).to be false
+      end
+    end
+  end
+end

data/spec/shoryuken/manager_spec.rb

@@ -15,7 +15,7 @@ RSpec.describe Shoryuken::Manager do
   let(:concurrency) { 1 }
   let(:executor) { Concurrent::ImmediateExecutor.new }
 
-  subject { Shoryuken::Manager.new(fetcher, polling_strategy, concurrency, executor) }
+  subject { Shoryuken::Manager.new('default', fetcher, polling_strategy, concurrency, executor) }
 
   before do
     allow(fetcher).to receive(:fetch).and_return([])
@@ -71,6 +71,11 @@ RSpec.describe Shoryuken::Manager do
 
       expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
       expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
+      expect(subject).to receive(:fire_event).with(:utilization_update,
+                                                   false,
+                                                   group: 'default',
+                                                   busy_processors: 1,
+                                                   max_processors: 1)
       expect(Shoryuken::Processor).to receive(:process).with(q, message)
       expect(Shoryuken.logger).to receive(:info).never
 
@@ -99,6 +104,11 @@ RSpec.describe Shoryuken::Manager do
         q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
 
         expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
+        expect(subject).to receive(:fire_event).with(:utilization_update,
+                                                     false,
+                                                     group: 'default',
+                                                     busy_processors: 1,
+                                                     max_processors: 1)
         expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
         allow(subject).to receive(:batched_queue?).with(q).and_return(true)
         expect(Shoryuken::Processor).to receive(:process).with(q, messages)
@@ -163,4 +173,26 @@ RSpec.describe Shoryuken::Manager do
       end
     end
   end
+
+  describe '#running?' do
+    context 'when the executor is running' do
+      before do
+        allow(executor).to receive(:running?).and_return(true)
+      end
+
+      it 'returns true' do
+        expect(subject.running?).to be true
+      end
+    end
+
+    context 'when the executor is not running' do
+      before do
+        allow(executor).to receive(:running?).and_return(false)
+      end
+
+      it 'returns false' do
+        expect(subject.running?).to be false
+      end
+    end
+  end
 end

data/spec/shoryuken_spec.rb

@@ -1,4 +1,13 @@
 require 'spec_helper'
 
 RSpec.describe Shoryuken do
+  describe '.healthy?' do
+    before do
+      allow(Shoryuken::Runner).to receive(:instance).and_return(double(:instance, healthy?: :some_result))
+    end
+
+    it 'delegates to the runner instance' do
+      expect(described_class.healthy?).to eq(:some_result)
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 5.2.1
+  version: 5.3.1
 platform: ruby
 authors:
 - Pablo Cantero
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-07 00:00:00.000000000 Z
+date: 2022-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv
@@ -112,7 +112,6 @@ files:
 - Appraisals
 - CHANGELOG.md
 - Gemfile
-- Gemfile.aws-sdk-core-v2
 - LICENSE
 - README.md
 - Rakefile
@@ -122,6 +121,7 @@ files:
 - examples/bootstrap_queues.rb
 - examples/default_worker.rb
 - gemfiles/.gitignore
+- gemfiles/aws_sdk_core_2.gemfile
 - gemfiles/rails_4_2.gemfile
 - gemfiles/rails_5_2.gemfile
 - gemfiles/rails_6_0.gemfile
@@ -174,6 +174,7 @@ files:
 - spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb
 - spec/shoryuken/extensions/active_job_wrapper_spec.rb
 - spec/shoryuken/fetcher_spec.rb
+- spec/shoryuken/launcher_spec.rb
 - spec/shoryuken/manager_spec.rb
 - spec/shoryuken/middleware/chain_spec.rb
 - spec/shoryuken/middleware/server/auto_delete_spec.rb
@@ -198,7 +199,7 @@ homepage: https://github.com/phstc/shoryuken
 licenses:
 - LGPL-3.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -213,8 +214,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
-signing_key:
+rubygems_version: 3.0.3.1
+signing_key: 
 specification_version: 4
 summary: Shoryuken is a super efficient AWS SQS thread based message processor
 test_files:
@@ -231,6 +232,7 @@ test_files:
 - spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb
 - spec/shoryuken/extensions/active_job_wrapper_spec.rb
 - spec/shoryuken/fetcher_spec.rb
+- spec/shoryuken/launcher_spec.rb
 - spec/shoryuken/manager_spec.rb
 - spec/shoryuken/middleware/chain_spec.rb
 - spec/shoryuken/middleware/server/auto_delete_spec.rb

data/Gemfile.aws-sdk-core-v2

@@ -1,13 +0,0 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in shoryuken.gemspec
-gemspec
-
-group :test do
-  gem 'activejob', '~> 4'
-  gem 'aws-sdk-core', '~> 2'
-  gem 'codeclimate-test-reporter', require: nil
-  gem 'httparty'
-  gem 'multi_xml'
-  gem 'simplecov'
-end