shoryuken 5.3.0 → 6.2.1

data/spec/shoryuken/environment_loader_spec.rb

@@ -4,11 +4,41 @@ require 'active_job'
 RSpec.describe Shoryuken::EnvironmentLoader do
   subject { described_class.new({}) }
 
-  describe '#parse_queues loads default queues' do
+  describe '#load' do
     before do
+      Shoryuken.groups.clear
+      # See issue: https://stackoverflow.com/a/63699568 for stubbing AWS errors
+      allow(Shoryuken::Client)
+        .to receive(:queues)
+        .with('stubbed_queue')
+        .and_raise(Aws::SQS::Errors::NonExistentQueue.new(nil, nil))
       allow(subject).to receive(:load_rails)
       allow(subject).to receive(:prefix_active_job_queue_names)
       allow(subject).to receive(:require_workers)
+      allow(subject).to receive(:validate_workers)
+      allow(subject).to receive(:patch_deprecated_workers)
+      Shoryuken.options[:groups] = [['custom', { queues: ['stubbed_queue'] }]]
+    end
+
+    context "when given queues don't exist" do
+      specify do
+        expect { subject.load }.to raise_error(
+          ArgumentError,
+          <<-MSG.gsub(/^\s+/, '')
+            The specified queue(s) stubbed_queue do not exist.
+            Try 'shoryuken sqs create QUEUE-NAME' for creating a queue with default settings.
+            It's also possible that you don't have permission to access the specified queues.
+          MSG
+        )
+      end
+    end
+  end
+
+  describe '#parse_queues loads default queues' do
+    before do
+      allow(subject).to receive(:initialize_rails)
+      allow(subject).to receive(:prefix_active_job_queue_names)
+      allow(subject).to receive(:require_workers)
       allow(subject).to receive(:validate_queues)
       allow(subject).to receive(:validate_workers)
       allow(subject).to receive(:patch_deprecated_workers)
@@ -24,7 +54,7 @@ RSpec.describe Shoryuken::EnvironmentLoader do
 
   describe '#parse_queues includes delay per groups' do
     before do
-      allow(subject).to receive(:load_rails)
+      allow(subject).to receive(:initialize_rails)
       allow(subject).to receive(:prefix_active_job_queue_names)
       allow(subject).to receive(:require_workers)
       allow(subject).to receive(:validate_queues)
@@ -34,7 +64,7 @@ RSpec.describe Shoryuken::EnvironmentLoader do
 
     specify do
       Shoryuken.options[:queues] = ['queue1', 'queue2'] # default queues
-      Shoryuken.options[:groups] = [[ 'custom', { queues: ['queue3'], delay: 25 }]]
+      Shoryuken.options[:groups] = [['custom', { queues: ['queue3'], delay: 25 }]]
       subject.load
 
       expect(Shoryuken.groups['default'][:queues]).to eq(%w[queue1 queue2])
@@ -44,10 +74,9 @@ RSpec.describe Shoryuken::EnvironmentLoader do
     end
   end
 
-
   describe '#prefix_active_job_queue_names' do
     before do
-      allow(subject).to receive(:load_rails)
+      allow(subject).to receive(:initialize_rails)
       allow(subject).to receive(:require_workers)
       allow(subject).to receive(:validate_queues)
       allow(subject).to receive(:validate_workers)
@@ -76,7 +105,7 @@ RSpec.describe Shoryuken::EnvironmentLoader do
 
     it 'does not prefix url-based queues' do
       Shoryuken.options[:queues] = ['https://example.com/test_queue1']
-      Shoryuken.options[:groups] = {'group1' => {queues: ['https://example.com/test_group1_queue1']}}
+      Shoryuken.options[:groups] = { 'group1' => { queues: ['https://example.com/test_group1_queue1'] } }
 
       subject.load
 
@@ -86,7 +115,7 @@ RSpec.describe Shoryuken::EnvironmentLoader do
 
     it 'does not prefix arn-based queues' do
       Shoryuken.options[:queues] = ['arn:aws:sqs:fake-region-1:1234:test_queue1']
-      Shoryuken.options[:groups] = {'group1' => {queues: ['arn:aws:sqs:fake-region-1:1234:test_group1_queue1']}}
+      Shoryuken.options[:groups] = { 'group1' => { queues: ['arn:aws:sqs:fake-region-1:1234:test_group1_queue1'] } }
 
       subject.load
 
@@ -94,9 +123,11 @@ RSpec.describe Shoryuken::EnvironmentLoader do
       expect(Shoryuken.groups['group1'][:queues]).to(eq(['arn:aws:sqs:fake-region-1:1234:test_group1_queue1']))
     end
   end
+
   describe "#setup_options" do
-    let (:cli_queues) { { "queue1"=> 10, "queue2" => 20 } }
-    let (:config_queues) { [["queue1", 8], ["queue2", 4]] }
+    let(:cli_queues) { { "queue1" => 10, "queue2" => 20 } }
+    let(:config_queues) { [["queue1", 8], ["queue2", 4]] }
+
     context "when given queues through config and CLI" do
       specify do
         allow_any_instance_of(Shoryuken::EnvironmentLoader).to receive(:config_file_options).and_return({ queues: config_queues })
@@ -104,6 +135,7 @@ RSpec.describe Shoryuken::EnvironmentLoader do
         expect(Shoryuken.options[:queues]).to eq(cli_queues)
       end
     end
+
     context "when given queues through config only" do
       specify do
         allow_any_instance_of(Shoryuken::EnvironmentLoader).to receive(:config_file_options).and_return({ queues: config_queues })
@@ -111,6 +143,7 @@ RSpec.describe Shoryuken::EnvironmentLoader do
         expect(Shoryuken.options[:queues]).to eq(config_queues)
       end
     end
+
     context "when given queues through CLI only" do
       specify do
         Shoryuken::EnvironmentLoader.setup_options(queues: cli_queues)

data/spec/shoryuken/extensions/active_job_base_spec.rb

@@ -18,7 +18,7 @@ RSpec.describe ActiveJob::Base do
   end
 
   describe '#perform_now' do
-    it 'allows keyward args' do
+    it 'allows keyword args' do
       collaborator = double 'worker collaborator'
       subject.send(:define_method, :perform) do |**kwargs|
         collaborator.foo(**kwargs)

data/spec/shoryuken/fetcher_spec.rb

@@ -28,12 +28,12 @@ RSpec.describe Shoryuken::Fetcher do
 
       Shoryuken.sqs_client_receive_message_opts[group] = { wait_time_seconds: 10 }
 
-      expect(queue).to receive(:receive_messages).with(
+      expect(queue).to receive(:receive_messages).with({
         wait_time_seconds: 10,
         max_number_of_messages: limit,
         message_attribute_names: ['All'],
         attribute_names: ['All']
-      ).and_return([])
+      }).and_return([])
 
       subject.fetch(queue_config, limit)
     end
@@ -62,11 +62,11 @@ RSpec.describe Shoryuken::Fetcher do
 
         Shoryuken.sqs_client_receive_message_opts[queue_name] = { max_number_of_messages: 1 }
 
-        expect(queue).to receive(:receive_messages).with(
+        expect(queue).to receive(:receive_messages).with({
           max_number_of_messages: 1,
           message_attribute_names: ['All'],
           attribute_names: ['All']
-        ).and_return([])
+        }).and_return([])
 
         subject.fetch(queue_config, limit)
       end
@@ -78,11 +78,11 @@ RSpec.describe Shoryuken::Fetcher do
 
         Shoryuken.sqs_client_receive_message_opts[queue_name] = { max_number_of_messages: 20 }
 
-        expect(queue).to receive(:receive_messages).with(
+        expect(queue).to receive(:receive_messages).with({
           max_number_of_messages: limit,
           message_attribute_names: ['All'],
           attribute_names: ['All']
-        ).and_return([])
+        }).and_return([])
 
         subject.fetch(queue_config, limit)
       end
@@ -93,9 +93,9 @@ RSpec.describe Shoryuken::Fetcher do
 
       specify do
         allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
-        expect(queue).to receive(:receive_messages).with(
+        expect(queue).to receive(:receive_messages).with({
           max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']
-        ).and_return([])
+        }).and_return([])
 
         subject.fetch(queue_config, limit)
       end
@@ -109,9 +109,9 @@ RSpec.describe Shoryuken::Fetcher do
         # see https://github.com/phstc/shoryuken/pull/530
 
         allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
-        expect(queue).to receive(:receive_messages).with(
+        expect(queue).to receive(:receive_messages).with({
           max_number_of_messages: 1, attribute_names: ['All'], message_attribute_names: ['All']
-        ).and_return([])
+        }).and_return([])
 
         subject.fetch(queue_config, limit)
       end
@@ -123,9 +123,9 @@ RSpec.describe Shoryuken::Fetcher do
           allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
           allow(Shoryuken.worker_registry).to receive(:batch_receive_messages?).with(queue.name).and_return(true)
 
-          expect(queue).to receive(:receive_messages).with(
+          expect(queue).to receive(:receive_messages).with({
             max_number_of_messages: limit, attribute_names: ['All'], message_attribute_names: ['All']
-          ).and_return([])
+          }).and_return([])
 
           subject.fetch(queue_config, limit)
         end

data/spec/shoryuken/launcher_spec.rb

@@ -57,4 +57,49 @@ RSpec.describe Shoryuken::Launcher do
       end
     end
   end
-end
+
+  describe '#stop' do
+    before do
+      allow(first_group_manager).to receive(:stop_new_dispatching)
+      allow(first_group_manager).to receive(:await_dispatching_in_progress)
+      allow(second_group_manager).to receive(:stop_new_dispatching)
+      allow(second_group_manager).to receive(:await_dispatching_in_progress)
+    end
+
+    it 'fires quiet, shutdown and stopped event' do
+      allow(subject).to receive(:fire_event)
+      subject.stop
+      expect(subject).to have_received(:fire_event).with(:quiet, true)
+      expect(subject).to have_received(:fire_event).with(:shutdown, true)
+      expect(subject).to have_received(:fire_event).with(:stopped)
+    end
+
+    it 'stops the managers' do
+      subject.stop
+      expect(first_group_manager).to have_received(:stop_new_dispatching)
+      expect(second_group_manager).to have_received(:stop_new_dispatching)
+    end
+  end
+
+  describe '#stop!' do
+    before do
+      allow(first_group_manager).to receive(:stop_new_dispatching)
+      allow(first_group_manager).to receive(:await_dispatching_in_progress)
+      allow(second_group_manager).to receive(:stop_new_dispatching)
+      allow(second_group_manager).to receive(:await_dispatching_in_progress)
+    end
+
+    it 'fires shutdown and stopped event' do
+      allow(subject).to receive(:fire_event)
+      subject.stop!
+      expect(subject).to have_received(:fire_event).with(:shutdown, true)
+      expect(subject).to have_received(:fire_event).with(:stopped)
+    end
+
+    it 'stops the managers' do
+      subject.stop!
+      expect(first_group_manager).to have_received(:stop_new_dispatching)
+      expect(second_group_manager).to have_received(:stop_new_dispatching)
+    end
+  end
+end

data/spec/shoryuken/manager_spec.rb

@@ -73,9 +73,11 @@ RSpec.describe Shoryuken::Manager do
       expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
       expect(subject).to receive(:fire_event).with(:utilization_update,
                                                    false,
-                                                   group: 'default',
-                                                   busy_processors: 1,
-                                                   max_processors: 1)
+                                                   {
+                                                     group: 'default',
+                                                     busy_processors: 1,
+                                                     max_processors: 1
+                                                   })
       expect(Shoryuken::Processor).to receive(:process).with(q, message)
       expect(Shoryuken.logger).to receive(:info).never
 
@@ -106,9 +108,11 @@ RSpec.describe Shoryuken::Manager do
         expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
         expect(subject).to receive(:fire_event).with(:utilization_update,
                                                      false,
-                                                     group: 'default',
-                                                     busy_processors: 1,
-                                                     max_processors: 1)
+                                                     {
+                                                       group: 'default',
+                                                       busy_processors: 1,
+                                                       max_processors: 1
+                                                     })
         expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
         allow(subject).to receive(:batched_queue?).with(q).and_return(true)
         expect(Shoryuken::Processor).to receive(:process).with(q, messages)

data/spec/shoryuken/polling/weighted_round_robin_spec.rb

@@ -106,12 +106,37 @@ RSpec.describe Shoryuken::Polling::WeightedRoundRobin do
   end
 
   describe '#message_processed' do
-    it 'removes paused queue, adds to active queues' do
-      strategy = Shoryuken::Polling::WeightedRoundRobin.new([queue1, queue2])
-      strategy.send(:pause, queue1)
-      expect(strategy.active_queues).to eq([[queue2, 1]])
-      strategy.message_processed(queue1)
-      expect(strategy.active_queues).to eq([[queue2, 1], [queue1, 1]])
+    it 'removes delay from paused queue' do
+      queues << queue1
+      queues << queue2
+
+      expect(subject.next_queue).to eq(queue1)
+      subject.messages_found(queue1, 0) # pauses queue1
+
+      expect(subject.active_queues).to eq([[queue2, 1]])
+
+      subject.message_processed(queue1) # marks queue1 to be unpaused
+
+      expect(subject.next_queue).to eq(queue2) # implicitly unpauses queue1
+      expect(subject.active_queues).to eq([[queue1, 1], [queue2, 1]])
+    end
+
+    it 'preserves weight of queues when unpausing' do
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.next_queue).to eq(queue1)
+      subject.messages_found(queue1, 1)
+
+      expect(subject.next_queue).to eq(queue2)
+      subject.messages_found(queue2, 0) # pauses queue2
+
+      expect(subject.active_queues).to eq([[queue1, 2]])
+      subject.message_processed(queue2) # marks queue2 to be unpaused
+
+      expect(subject.next_queue).to eq(queue1) # implicitly unpauses queue2
+      expect(subject.active_queues).to eq([[queue1, 2], [queue2, 1]])
     end
   end
 end

data/spec/shoryuken/processor_spec.rb

@@ -159,5 +159,43 @@ RSpec.describe Shoryuken::Processor do
         subject.process
       end
     end
+
+    context 'when specifying a reloader' do
+      before do
+        Shoryuken.reloader = proc do |_|
+          TestWorker.new.called
+        end
+      end
+
+      after do
+        Shoryuken.reloader = proc { |&block| block.call }
+      end
+
+      context 'when reloader is enabled' do
+        before do
+          Shoryuken.enable_reloading = true
+        end
+
+        it 'wraps execution in reloader' do
+          expect_any_instance_of(TestWorker).to receive(:called)
+          expect_any_instance_of(TestWorker).to_not receive(:perform)
+
+          subject.process
+        end
+
+        after do
+          Shoryuken.enable_reloading = false
+        end
+      end
+
+      context 'when reloader is disabled' do
+        it 'does not wrap execution in reloader' do
+          expect_any_instance_of(TestWorker).to_not receive(:called)
+          expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
+
+          subject.process
+        end
+      end
+    end
   end
 end

data/spec/shoryuken/queue_spec.rb

@@ -80,7 +80,8 @@ RSpec.describe Shoryuken::Queue do
     end
 
     it 'deletes' do
-      expect(sqs).to receive(:delete_message_batch).with(entries: entries, queue_url: queue_url).and_return(double(failed: []))
+      expect(sqs).to receive(:delete_message_batch).with({ entries: entries,
+                                                           queue_url: queue_url }).and_return(double(failed: []))
 
       subject.delete_messages(entries: entries)
     end
@@ -91,7 +92,8 @@ RSpec.describe Shoryuken::Queue do
         logger = double 'Logger'
 
         expect(sqs).to(
-          receive(:delete_message_batch).with(entries: entries, queue_url: queue_url).and_return(double(failed: [failure]))
+          receive(:delete_message_batch).with({ entries: entries,
+                                                queue_url: queue_url }).and_return(double(failed: [failure]))
         )
         expect(subject).to receive(:logger).and_return(logger)
         expect(logger).to receive(:error)
@@ -157,7 +159,8 @@ RSpec.describe Shoryuken::Queue do
     it 'accepts SQS request parameters' do
       # https://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#send_message_batch-instance_method
       expect(sqs).to(
-        receive(:send_message_batch).with(hash_including(entries: [{ id: '0', message_body: 'msg1' }, { id: '1', message_body: 'msg2' }]))
+        receive(:send_message_batch).with(hash_including(entries: [{ id: '0', message_body: 'msg1' },
+                                                                   { id: '1', message_body: 'msg2' }]))
       )
 
       subject.send_messages(entries: [{ id: '0', message_body: 'msg1' }, { id: '1', message_body: 'msg2' }])
@@ -286,7 +289,8 @@ RSpec.describe Shoryuken::Queue do
         Shoryuken.cache_visibility_timeout = false
 
         expect(sqs).to(
-          receive(:get_queue_attributes).with(queue_url: queue_url, attribute_names: ['All']).and_return(attribute_response).exactly(3).times
+          receive(:get_queue_attributes).with(queue_url: queue_url,
+                                              attribute_names: ['All']).and_return(attribute_response).exactly(3).times
         )
         expect(subject.visibility_timeout).to eq(30)
         expect(subject.visibility_timeout).to eq(30)
@@ -299,7 +303,8 @@ RSpec.describe Shoryuken::Queue do
         Shoryuken.cache_visibility_timeout = true
 
         expect(sqs).to(
-          receive(:get_queue_attributes).with(queue_url: queue_url, attribute_names: ['All']).and_return(attribute_response).once
+          receive(:get_queue_attributes).with({ queue_url: queue_url,
+                                                attribute_names: ['All'] }).and_return(attribute_response).once
         )
         expect(subject.visibility_timeout).to eq(30)
         expect(subject.visibility_timeout).to eq(30)

data/spec/shoryuken/util_spec.rb

@@ -17,14 +17,34 @@ describe 'Shoryuken::Util' do
 
   describe '#worker_name' do
     let(:sqs_msg) do
-      double Shoryuken::Message, message_id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e', message_attributes: {}
+      double Shoryuken::Message, message_id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e', message_attributes: message_attributes
     end
 
-    it 'returns Shoryuken worker name' do
-      expect(subject.worker_name(TestWorker, sqs_msg)).to eq 'TestWorker'
+    context 'when has integration with ActiveJob' do
+      before do
+        allow(Shoryuken).to receive(:active_job?).and_return(true)
+      end
+
+      let(:message_attributes) do
+        { 'shoryuken_class' => { string_value: ActiveJob::QueueAdapters::ShoryukenAdapter::JobWrapper.to_s } }
+      end
+
+      let(:body) do
+        { 'job_class' => 'TestJob' }
+      end
+
+      it 'returns ActiveJob worker name' do
+        expect(subject.worker_name(TestWorker, sqs_msg, body)).to eq 'ActiveJob/TestJob'
+      end
     end
 
-    it 'returns ActiveJob worker name'
+    context 'when has not integration with ActiveJob' do
+      let(:message_attributes) { {} }
+
+      it 'returns Shoryuken worker name' do
+        expect(subject.worker_name(TestWorker, sqs_msg)).to eq 'TestWorker'
+      end
+    end
   end
 
   describe '#fire_event' do

data/spec/shoryuken/worker/default_executor_spec.rb

@@ -10,16 +10,16 @@ RSpec.describe Shoryuken::Worker::DefaultExecutor do
 
   describe '.perform_in' do
     it 'delays a message' do
-      expect(sqs_queue).to receive(:send_message).with(
-        message_attributes: {
-          'shoryuken_class' => {
-            string_value: TestWorker.to_s,
-            data_type: 'String'
-          }
-        },
-        message_body: 'message',
-        delay_seconds: 60
-      )
+      expect(sqs_queue).to receive(:send_message).with({
+                                                         message_attributes: {
+                                                           'shoryuken_class' => {
+                                                             string_value: TestWorker.to_s,
+                                                             data_type: 'String'
+                                                           }
+                                                         },
+                                                         message_body: 'message',
+                                                         delay_seconds: 60
+                                                       })
 
       TestWorker.perform_in(60, 'message')
     end
@@ -33,16 +33,16 @@ RSpec.describe Shoryuken::Worker::DefaultExecutor do
 
   describe '.perform_at' do
     it 'delays a message' do
-      expect(sqs_queue).to receive(:send_message).with(
-        message_attributes: {
-          'shoryuken_class' => {
-            string_value: TestWorker.to_s,
-            data_type: 'String'
-          }
-        },
-        message_body: 'message',
-        delay_seconds: 60
-      )
+      expect(sqs_queue).to receive(:send_message).with({
+                                                         message_attributes: {
+                                                           'shoryuken_class' => {
+                                                             string_value: TestWorker.to_s,
+                                                             data_type: 'String'
+                                                           }
+                                                         },
+                                                         message_body: 'message',
+                                                         delay_seconds: 60
+                                                       })
 
       TestWorker.perform_in(Time.now + 60, 'message')
     end
@@ -56,30 +56,30 @@ RSpec.describe Shoryuken::Worker::DefaultExecutor do
 
   describe '.perform_async' do
     it 'enqueues a message' do
-      expect(sqs_queue).to receive(:send_message).with(
-        message_attributes: {
-          'shoryuken_class' => {
-            string_value: TestWorker.to_s,
-            data_type: 'String'
-          }
-        },
-        message_body: 'message'
-      )
+      expect(sqs_queue).to receive(:send_message).with({
+                                                         message_attributes: {
+                                                           'shoryuken_class' => {
+                                                             string_value: TestWorker.to_s,
+                                                             data_type: 'String'
+                                                           }
+                                                         },
+                                                         message_body: 'message'
+                                                       })
 
       TestWorker.perform_async('message')
     end
 
     it 'enqueues a message with options' do
-      expect(sqs_queue).to receive(:send_message).with(
-        delay_seconds: 60,
-        message_attributes: {
-          'shoryuken_class' => {
-            string_value: TestWorker.to_s,
-            data_type: 'String'
-          }
-        },
-        message_body: 'delayed message'
-      )
+      expect(sqs_queue).to receive(:send_message).with({
+                                                         delay_seconds: 60,
+                                                         message_attributes: {
+                                                           'shoryuken_class' => {
+                                                             string_value: TestWorker.to_s,
+                                                             data_type: 'String'
+                                                           }
+                                                         },
+                                                         message_body: 'delayed message'
+                                                       })
 
       TestWorker.perform_async('delayed message', delay_seconds: 60)
     end
@@ -89,15 +89,15 @@ RSpec.describe Shoryuken::Worker::DefaultExecutor do
 
       expect(Shoryuken::Client).to receive(:queues).with(new_queue).and_return(sqs_queue)
 
-      expect(sqs_queue).to receive(:send_message).with(
-        message_attributes: {
-          'shoryuken_class' => {
-            string_value: TestWorker.to_s,
-            data_type: 'String'
-          }
-        },
-        message_body: 'delayed message'
-      )
+      expect(sqs_queue).to receive(:send_message).with({
+                                                         message_attributes: {
+                                                           'shoryuken_class' => {
+                                                             string_value: TestWorker.to_s,
+                                                             data_type: 'String'
+                                                           }
+                                                         },
+                                                         message_body: 'delayed message'
+                                                       })
 
       TestWorker.perform_async('delayed message', queue: new_queue)
     end

data/spec/spec_helper.rb

@@ -46,6 +46,8 @@ RSpec.configure do |config|
     Shoryuken.options[:logfile]     = nil
     Shoryuken.options[:queues]      = nil
 
+    Shoryuken.options[:exception_handlers] = []
+
     TestWorker.get_shoryuken_options.clear
     TestWorker.get_shoryuken_options['queue'] = 'default'