shoryuken 5.2.1 → 5.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bfdadf90a17ca05fbad43a5d38efc22179415f06b661b587cf5b087d0c200174
-  data.tar.gz: ca814528eaef23167aca0003fa62e180944ce3bc686d54728c9c30f82732a385
+  metadata.gz: 0f2f8c9e8573b699391a74a70d26756e106ccf72b3df9a6fccc32b72d13ec0cd
+  data.tar.gz: 6a009d5a5e571da339d52a08f4ccda19f144ce06132f4a65e95fabc39aeffad4
 SHA512:
-  metadata.gz: 680c478f59a3feb96c2ed27c6d169c3ad78d37191acf374dff456365a12611b3dba4c0b077e29808d55867446ab2a5008eca6db11f0707aa1a38caea5561d8ee
-  data.tar.gz: d0e80fc907715d307d6657f6f507643f732c417dc59f888c7123288a416f662a458aa15878516c79b4b9730ef5a306ba33d778354fc8d5cd7cbe448fd76a6a9c
+  metadata.gz: e359eee09d82a917c4e13541a121c61f78587421a7e5977052552a4f18512eeb9a2eb720ebc8adf323d0a33fd979b37e9c059a95b5b180eda93cab5a3e034f6b
+  data.tar.gz: 5217287d2b7f66125852ac0b824353d934a2a8568e2dc07ca3c98f8991cad1a0dafbda87967a407bc5efaea62b1cafa85708e1ea17ccb659b2743878d079a267

data/.github/workflows/specs.yml

@@ -9,8 +9,8 @@ jobs:
     name: All Specs
     strategy:
       matrix:
-        ruby: ['2.4.4', '2.5.1', '2.6.3']
-        gemfile: ['Gemfile', 'Gemfile.aws-sdk-core-v2']
+        ruby: ['2.4', '2.5', '2.6', '2.7', '3.0']
+        gemfile: ['Gemfile', 'gemfiles/aws_sdk_core_2.gemfile']
     runs-on: ubuntu-20.04
     services:
       moto_sqs:

data/Appraisals

@@ -1,3 +1,11 @@
+appraise 'aws_sdk_core_2' do
+  group :test do
+    gem 'aws-sdk-core', '~> 2'
+    gem 'webrick' # required for Ruby 3
+    remove_gem 'aws-sdk-sqs'
+  end
+end
+
 appraise 'rails_4_2' do
   group :test do
     gem 'activejob', '~> 4.2'

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## [v5.2.3] - 2021-07-29
+
+- Fire new `:utilization_update` event any time a worker pool's utilization changes
+  - [#673](https://github.com/ruby-shoryuken/shoryuken/pull/673)
+
+## [v5.2.2] - 2021-06-22
+
+- When using ActiveJob queue name prefixing, avoid applying prefix to queues configured with a URL or ARN
+  - [#667](https://github.com/ruby-shoryuken/shoryuken/pull/667)
+
 ## [v5.2.1] - 2021-04-06
 
 - Reduce message batch sizes in `shoryuken sqs requeue` and `shoryuken sqs mv` commands

data/gemfiles/aws_sdk_core_2.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob"
+  gem "aws-sdk-core", "~> 2"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+  gem "webrick"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/lib/shoryuken/environment_loader.rb

@@ -80,6 +80,8 @@ module Shoryuken
     end
 
     def prefix_active_job_queue_name(queue_name, weight)
+      return [queue_name, weight] if queue_name.start_with?('https://', 'arn:')
+
       queue_name_prefix = ::ActiveJob::Base.queue_name_prefix
       queue_name_delimiter = ::ActiveJob::Base.queue_name_delimiter
 

data/lib/shoryuken/launcher.rb

@@ -71,6 +71,7 @@ module Shoryuken
     def create_managers
       Shoryuken.groups.map do |group, options|
         Shoryuken::Manager.new(
+          group,
           Shoryuken::Fetcher.new(group),
           Shoryuken.polling_strategy(group).new(options[:queues], Shoryuken.delay(group)),
           options[:concurrency],

data/lib/shoryuken/manager.rb

@@ -6,7 +6,8 @@ module Shoryuken
     # See https://github.com/phstc/shoryuken/issues/348#issuecomment-292847028
     MIN_DISPATCH_INTERVAL = 0.1
 
-    def initialize(fetcher, polling_strategy, concurrency, executor)
+    def initialize(group, fetcher, polling_strategy, concurrency, executor)
+      @group            = group
       @fetcher          = fetcher
       @polling_strategy = polling_strategy
       @max_processors   = concurrency
@@ -16,6 +17,7 @@ module Shoryuken
     end
 
     def start
+      fire_utilization_update_event
       dispatch_loop
     end
 
@@ -59,6 +61,8 @@ module Shoryuken
 
     def processor_done(queue)
       @busy_processors.decrement
+      fire_utilization_update_event
+
       client_queue = Shoryuken::Client.queues(queue)
       return unless client_queue.fifo?
       return unless @polling_strategy.respond_to?(:message_processed)
@@ -72,6 +76,7 @@ module Shoryuken
       logger.debug { "Assigning #{sqs_msg.message_id}" }
 
       @busy_processors.increment
+      fire_utilization_update_event
 
       Concurrent::Promise
         .execute(executor: @executor) { Processor.process(queue_name, sqs_msg) }
@@ -114,5 +119,13 @@ module Shoryuken
 
       @running.make_false
     end
+
+    def fire_utilization_update_event
+      fire_event :utilization_update, false, {
+        group: @group,
+        max_processors: @max_processors,
+        busy_processors: busy
+      }
+    end
   end
 end

data/lib/shoryuken/options.rb

@@ -9,6 +9,7 @@ module Shoryuken
       lifecycle_events: {
         startup: [],
         dispatch: [],
+        utilization_update: [],
         quiet: [],
         shutdown: []
       }

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '5.2.1'.freeze
+  VERSION = '5.2.3'.freeze
 end

data/spec/shoryuken/environment_loader_spec.rb

@@ -57,11 +57,11 @@ RSpec.describe Shoryuken::EnvironmentLoader do
       ActiveJob::Base.queue_name_delimiter = '_'
 
       allow(Shoryuken).to receive(:active_job?).and_return(true)
-    end
 
-    specify do
       Shoryuken.active_job_queue_name_prefixing = true
+    end
 
+    specify do
       Shoryuken.options[:queues] = ['queue1', ['queue2', 2]]
 
       Shoryuken.options[:groups] = {
@@ -73,6 +73,26 @@ RSpec.describe Shoryuken::EnvironmentLoader do
       expect(Shoryuken.groups['default'][:queues]).to eq(%w[test_queue1 test_queue2 test_queue2])
       expect(Shoryuken.groups['group1'][:queues]).to eq(%w[test_group1_queue1 test_group1_queue2])
     end
+
+    it 'does not prefix url-based queues' do
+      Shoryuken.options[:queues] = ['https://example.com/test_queue1']
+      Shoryuken.options[:groups] = {'group1' => {queues: ['https://example.com/test_group1_queue1']}}
+
+      subject.load
+
+      expect(Shoryuken.groups['default'][:queues]).to(eq(['https://example.com/test_queue1']))
+      expect(Shoryuken.groups['group1'][:queues]).to(eq(['https://example.com/test_group1_queue1']))
+    end
+
+    it 'does not prefix arn-based queues' do
+      Shoryuken.options[:queues] = ['arn:aws:sqs:fake-region-1:1234:test_queue1']
+      Shoryuken.options[:groups] = {'group1' => {queues: ['arn:aws:sqs:fake-region-1:1234:test_group1_queue1']}}
+
+      subject.load
+
+      expect(Shoryuken.groups['default'][:queues]).to(eq(['arn:aws:sqs:fake-region-1:1234:test_queue1']))
+      expect(Shoryuken.groups['group1'][:queues]).to(eq(['arn:aws:sqs:fake-region-1:1234:test_group1_queue1']))
+    end
   end
   describe "#setup_options" do
     let (:cli_queues) { { "queue1"=> 10, "queue2" => 20 } }

data/spec/shoryuken/manager_spec.rb

@@ -15,7 +15,7 @@ RSpec.describe Shoryuken::Manager do
   let(:concurrency) { 1 }
   let(:executor) { Concurrent::ImmediateExecutor.new }
 
-  subject { Shoryuken::Manager.new(fetcher, polling_strategy, concurrency, executor) }
+  subject { Shoryuken::Manager.new('default', fetcher, polling_strategy, concurrency, executor) }
 
   before do
     allow(fetcher).to receive(:fetch).and_return([])
@@ -71,6 +71,11 @@ RSpec.describe Shoryuken::Manager do
 
       expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
       expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
+      expect(subject).to receive(:fire_event).with(:utilization_update,
+                                                   false,
+                                                   group: 'default',
+                                                   busy_processors: 1,
+                                                   max_processors: 1)
       expect(Shoryuken::Processor).to receive(:process).with(q, message)
       expect(Shoryuken.logger).to receive(:info).never
 
@@ -99,6 +104,11 @@ RSpec.describe Shoryuken::Manager do
         q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
 
         expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
+        expect(subject).to receive(:fire_event).with(:utilization_update,
+                                                     false,
+                                                     group: 'default',
+                                                     busy_processors: 1,
+                                                     max_processors: 1)
         expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
         allow(subject).to receive(:batched_queue?).with(q).and_return(true)
         expect(Shoryuken::Processor).to receive(:process).with(q, messages)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 5.2.1
+  version: 5.2.3
 platform: ruby
 authors:
 - Pablo Cantero
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-07 00:00:00.000000000 Z
+date: 2021-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv
@@ -112,7 +112,6 @@ files:
 - Appraisals
 - CHANGELOG.md
 - Gemfile
-- Gemfile.aws-sdk-core-v2
 - LICENSE
 - README.md
 - Rakefile
@@ -122,6 +121,7 @@ files:
 - examples/bootstrap_queues.rb
 - examples/default_worker.rb
 - gemfiles/.gitignore
+- gemfiles/aws_sdk_core_2.gemfile
 - gemfiles/rails_4_2.gemfile
 - gemfiles/rails_5_2.gemfile
 - gemfiles/rails_6_0.gemfile

data/Gemfile.aws-sdk-core-v2

@@ -1,13 +0,0 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in shoryuken.gemspec
-gemspec
-
-group :test do
-  gem 'activejob', '~> 4'
-  gem 'aws-sdk-core', '~> 2'
-  gem 'codeclimate-test-reporter', require: nil
-  gem 'httparty'
-  gem 'multi_xml'
-  gem 'simplecov'
-end