shoryuken 2.0.11 → 3.0.6

data/spec/shoryuken/middleware/chain_spec.rb

@@ -44,28 +44,4 @@ describe Shoryuken::Middleware::Chain do
     expect(final_action).to eq nil
     expect(recorder).to eq []
   end
-
-  class DeprecatedMiddleware
-    def call(worker_instance, queue, sqs_msg)
-      @@success = true
-    end
-
-    def self.success?
-      !!@@success
-    end
-  end
-
-  it 'patches deprecated middleware' do
-    subject.clear
-
-    expect(Shoryuken.logger).to receive(:warn) do |&block|
-      expect(block.call).to eq('[DEPRECATION] DeprecatedMiddleware#call(worker_instance, queue, sqs_msg) is deprecated. Please use DeprecatedMiddleware#call(worker_instance, queue, sqs_msg, body)')
-    end
-
-    subject.add DeprecatedMiddleware
-
-    subject.invoke TestWorker, 'test', double('SQS msg', body: 'test'), 'test'
-
-    expect(DeprecatedMiddleware.success?).to eq true
-  end
 end

data/spec/shoryuken/middleware/server/auto_delete_spec.rb

@@ -55,8 +55,8 @@ describe Shoryuken::Middleware::Server::AutoDelete do
       expect(sqs_queue).to_not receive(:delete_messages)
 
       expect {
-        subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'Error' }
-      }.to raise_error(RuntimeError, 'Error')
+        subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' }
+      }.to raise_error('failed')
     end
   end
 end

data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Shoryuken::Middleware::Server::AutoExtendVisibility do
+RSpec.describe Shoryuken::Middleware::Server::AutoExtendVisibility do
   let(:queue) { 'default' }
   let(:visibility_timeout) { 3 }
   let(:extend_upfront) { 1 }
@@ -12,8 +12,6 @@ describe Shoryuken::Middleware::Server::AutoExtendVisibility do
 
   # We need to run our worker inside actor context.
   class Runner
-    include Celluloid
-
     def run_and_sleep(worker, queue, sqs_msg, interval)
       Shoryuken::Middleware::Server::AutoExtendVisibility.new.call(worker, queue, sqs_msg, sqs_msg.body) do
         sleep interval
@@ -34,6 +32,12 @@ describe Shoryuken::Middleware::Server::AutoExtendVisibility do
     stub_const('Shoryuken::Middleware::Server::AutoExtendVisibility::EXTEND_UPFRONT_SECONDS', extend_upfront)
   end
 
+  context 'when batch worker' do
+    it 'yields' do
+      expect { |b| subject.call(nil, nil, [], nil, &b) }.to yield_control
+    end
+  end
+
   it 'extends message visibility if jobs takes a long time' do
     TestWorker.get_shoryuken_options['auto_visibility_timeout'] = true
 

data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb

@@ -1,6 +1,7 @@
 require 'spec_helper'
 
-describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
+# rubocop:disable Metrics/BlockLength, Metrics/BlockDelimiters
+RSpec.describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
   let(:queue)     { 'default' }
   let(:sqs_queue) { double Shoryuken::Queue }
   let(:sqs_msg)   { double Shoryuken::Message, queue_url: queue, body: 'test', receipt_handle: SecureRandom.uuid,
@@ -10,8 +11,14 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
     allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
   end
 
-  context 'when a job succeeds' do
-    it 'does not retry the job' do
+  context 'when batch worker' do
+    it 'yields' do
+      expect { |b| subject.call(nil, nil, [], nil, &b) }.to yield_control
+    end
+  end
+
+  context 'when no exception' do
+    it 'does not retry' do
       TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
 
       expect(sqs_msg).not_to receive(:change_visibility)
@@ -20,62 +27,78 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
     end
   end
 
-  context 'when a job throws an exception' do
+  context 'when an error' do
+    context "and retry_intervals isn't set" do
+      it 'does not retry' do
+        expect(sqs_msg).not_to receive(:change_visibility)
 
-    it 'does not retry the job by default' do
-      expect(sqs_msg).not_to receive(:change_visibility)
+        expect {
+          subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'Error' }
+        }.to raise_error(RuntimeError, 'Error')
+      end
+    end
 
-      expect {
-        subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'Error' }
-      }.to raise_error(RuntimeError, 'Error')
+    context 'and retry_intervals is a lambda' do
+      it 'retries' do
+        TestWorker.get_shoryuken_options['retry_intervals'] = ->(_attempts) { 500 }
+
+        allow(sqs_msg).to receive(:queue) { sqs_queue }
+        expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 500)
+
+        expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
+      end
     end
 
-    it 'does not retry the job if :retry_intervals is empty' do
-      TestWorker.get_shoryuken_options['retry_intervals'] = []
+    context 'and retry_intervals is empty' do
+      it 'does not retry' do
+        TestWorker.get_shoryuken_options['retry_intervals'] = []
 
-      expect(sqs_msg).not_to receive(:change_visibility)
+        expect(sqs_msg).not_to receive(:change_visibility)
 
-      expect {
-        subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'Error' }
-      }.to raise_error(RuntimeError, 'Error')
+        expect {
+          subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'Error' }
+        }.to raise_error(RuntimeError, 'Error')
+      end
     end
 
-    it 'retries the job if :retry_intervals is non-empty' do
+    it 'uses first interval ' do
       TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
 
-      allow(sqs_msg).to receive(:queue){ sqs_queue }
+      allow(sqs_msg).to receive(:queue) { sqs_queue }
       expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 300)
 
-      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
+      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
     end
 
-    it 'retries the job with exponential backoff' do
+    it 'uses matching interval' do
       TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
 
-      allow(sqs_msg).to receive(:attributes){ {'ApproximateReceiveCount' => 2 } }
-      allow(sqs_msg).to receive(:queue){ sqs_queue }
+      allow(sqs_msg).to receive(:attributes) { { 'ApproximateReceiveCount' => 2 } }
+      allow(sqs_msg).to receive(:queue) { sqs_queue }
       expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
 
-      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
+      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
     end
 
-    it 'uses the last retry interval when :receive_count exceeds the size of :retry_intervals' do
-      TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
+    context 'when attempts exceeds retry_intervals' do
+      it 'uses last interval' do
+        TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
 
-      allow(sqs_msg).to receive(:attributes){ {'ApproximateReceiveCount' => 3 } }
-      allow(sqs_msg).to receive(:queue){ sqs_queue }
-      expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
+        allow(sqs_msg).to receive(:attributes) { { 'ApproximateReceiveCount' => 3 } }
+        allow(sqs_msg).to receive(:queue) { sqs_queue }
+        expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
 
-      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
+        expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
+      end
     end
 
-    it 'limits the visibility timeout to 12 hours from receipt of message' do
-      TestWorker.get_shoryuken_options['retry_intervals'] = [86400]
+    it 'limits the visibility timeout to 12 hours' do
+      TestWorker.get_shoryuken_options['retry_intervals'] = [86_400]
 
-      allow(sqs_msg).to receive(:queue){ sqs_queue }
-      expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 43198)
+      allow(sqs_msg).to receive(:queue) { sqs_queue }
+      expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 43_198)
 
-      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
+      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
     end
   end
 end

data/spec/shoryuken/polling_spec.rb

@@ -0,0 +1,239 @@
+require 'spec_helper'
+require 'shoryuken/polling'
+
+describe Shoryuken::Polling::WeightedRoundRobin do
+  let(:queue1) { 'shoryuken' }
+  let(:queue2) { 'uppercut' }
+  let(:queues) { Array.new }
+  subject { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
+
+  describe '#next_queue' do
+    it 'cycles' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      expect(subject.next_queue).to eq(queue1)
+    end
+
+    it 'returns nil if there are no active queues' do
+      expect(subject.next_queue).to eq(nil)
+    end
+
+    it 'unpauses queues whose pause is expired' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      allow(subject).to receive(:delay).and_return(10)
+
+      now = Time.now
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the first queue
+      subject.messages_found(queue1, 0)
+      expect(subject.next_queue).to eq(queue2)
+
+      now += 5
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the second queue
+      subject.messages_found(queue2, 0)
+      expect(subject.next_queue).to eq(nil)
+
+      # queue1 should be unpaused now
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue1)
+
+      # queue1 should be unpaused and added to the end of queues now
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+    end
+  end
+
+  describe '#messages_found' do
+    it 'pauses a queue if there are no messages found' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject).to receive(:pause).with(queue1).and_call_original
+      subject.messages_found(queue1, 0)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue2])
+    end
+
+    it 'increased the weight if message is found' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2])
+      subject.messages_found(queue1, 1)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
+    end
+
+    it 'respects the maximum queue weight' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      subject.messages_found(queue1, 1)
+      subject.messages_found(queue1, 1)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
+    end
+  end
+end
+
+describe Shoryuken::Polling::StrictPriority do
+  let(:queue1) { 'shoryuken' }
+  let(:queue2) { 'uppercut' }
+  let(:queue3) { 'other' }
+  let(:queues) { Array.new }
+  subject { Shoryuken::Polling::StrictPriority.new(queues) }
+
+  describe '#next_queue' do
+    it 'cycles when declared desc' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+    end
+
+    it 'cycles when declared asc' do
+      # [uppercut,  1]
+      # [shoryuken, 2]
+      queues << queue2
+      queues << queue1
+      queues << queue1
+
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+    end
+
+    it 'returns nil if there are no active queues' do
+      expect(subject.next_queue).to eq(nil)
+    end
+
+    it 'unpauses queues whose pause is expired' do
+      # [shoryuken, 3]
+      # [uppercut,  2]
+      # [other,     1]
+      queues << queue1
+      queues << queue1
+      queues << queue1
+      queues << queue2
+      queues << queue2
+      queues << queue3
+
+      allow(subject).to receive(:delay).and_return(10)
+
+      now = Time.now
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the second queue, see it loop between 1 and 3
+      subject.messages_found(queue2, 0)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue3)
+      expect(subject.next_queue).to eq(queue1)
+
+      now += 5
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the first queue, see it repeat 3
+      subject.messages_found(queue1, 0)
+      expect(subject.next_queue).to eq(queue3)
+      expect(subject.next_queue).to eq(queue3)
+
+      # pause the third queue, see it have nothing
+      subject.messages_found(queue3, 0)
+      expect(subject.next_queue).to eq(nil)
+
+      # unpause queue 2
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue2)
+
+      # unpause queues 1 and 3
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      expect(subject.next_queue).to eq(queue3)
+    end
+  end
+
+  describe '#messages_found' do
+    it 'pauses a queue if there are no messages found' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.active_queues).to eq([[queue1, 2], [queue2, 1]])
+      expect(subject).to receive(:pause).with(queue1).and_call_original
+      subject.messages_found(queue1, 0)
+      expect(subject.active_queues).to eq([[queue2, 1]])
+    end
+
+    it 'continues to queue the highest priority queue if messages are found' do
+      # [shoryuken, 3]
+      # [uppercut,  2]
+      # [other,     1]
+      queues << queue1
+      queues << queue1
+      queues << queue1
+      queues << queue2
+      queues << queue2
+      queues << queue3
+
+      expect(subject.next_queue).to eq(queue1)
+      subject.messages_found(queue1, 1)
+      expect(subject.next_queue).to eq(queue1)
+      subject.messages_found(queue1, 1)
+      expect(subject.next_queue).to eq(queue1)
+    end
+
+    it 'resets the priorities if messages are found part way' do
+      # [shoryuken, 3]
+      # [uppercut,  2]
+      # [other,     1]
+      queues << queue1
+      queues << queue1
+      queues << queue1
+      queues << queue2
+      queues << queue2
+      queues << queue3
+
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      subject.messages_found(queue2, 1)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      expect(subject.next_queue).to eq(queue3)
+    end
+  end
+end

data/spec/shoryuken/processor_spec.rb

@@ -99,6 +99,7 @@ RSpec.describe Shoryuken::Processor do
       end
 
       it 'logs the error' do
+        expect(manager).to receive(:processor_failed)
         expect(subject.logger).to receive(:error) do |&block|
           expect(block.call).
             to include("unexpected token at 'invalid json'\nbody_parser: json\nsqs_msg.body: invalid json")
@@ -108,6 +109,7 @@ RSpec.describe Shoryuken::Processor do
       end
 
       it 're raises the error' do
+        expect(manager).to receive(:processor_failed)
         expect { subject.process(queue, sqs_msg) }.
           to raise_error(JSON::ParserError, /unexpected token at 'invalid json'/)
       end
@@ -169,7 +171,7 @@ RSpec.describe Shoryuken::Processor do
         end
 
         it 'invokes middleware' do
-          expect(manager).to receive(:processor_done).with(queue, subject)
+          expect(manager).to receive(:processor_done).with(queue)
 
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:called).with(sqs_msg, queue)
@@ -199,7 +201,7 @@ RSpec.describe Shoryuken::Processor do
         end
 
         it "doesn't invoke middleware" do
-          expect(manager).to receive(:processor_done).with(queue, subject)
+          expect(manager).to receive(:processor_done).with(queue)
 
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to_not receive(:called).with(sqs_msg, queue)
@@ -212,7 +214,7 @@ RSpec.describe Shoryuken::Processor do
     it 'performs with delete' do
       TestWorker.get_shoryuken_options['auto_delete'] = true
 
-      expect(manager).to receive(:processor_done).with(queue, subject)
+      expect(manager).to receive(:processor_done).with(queue)
 
       expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
 
@@ -224,7 +226,7 @@ RSpec.describe Shoryuken::Processor do
     it 'performs without delete' do
       TestWorker.get_shoryuken_options['auto_delete'] = false
 
-      expect(manager).to receive(:processor_done).with(queue, subject)
+      expect(manager).to receive(:processor_done).with(queue)
 
       expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
 
@@ -249,7 +251,7 @@ RSpec.describe Shoryuken::Processor do
       it 'performs without delete' do
         Shoryuken.worker_registry.clear # unregister TestWorker
 
-        expect(manager).to receive(:processor_done).with(queue, subject)
+        expect(manager).to receive(:processor_done).with(queue)
 
         expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)