shoryuken 3.0.4 → 3.1.0

data/spec/shoryuken/options_spec.rb

@@ -0,0 +1,100 @@
+require 'spec_helper'
+
+RSpec.describe Shoryuken::Options do
+  describe '.add_group' do
+    before do
+      Shoryuken.groups.clear
+      Shoryuken.add_group('group1', 25)
+      Shoryuken.add_group('group2', 25)
+    end
+
+    specify do
+      described_class.add_queue('queue1', 1, 'group1')
+      described_class.add_queue('queue2', 2, 'group2')
+
+      expect(described_class.groups['group1'][:queues]).to eq(%w(queue1))
+      expect(described_class.groups['group2'][:queues]).to eq(%w(queue2 queue2))
+    end
+  end
+
+  describe '.ungrouped_queues' do
+    before do
+      Shoryuken.groups.clear
+      Shoryuken.add_group('group1', 25)
+      Shoryuken.add_group('group2', 25)
+    end
+
+    specify do
+      described_class.add_queue('queue1', 1, 'group1')
+      described_class.add_queue('queue2', 2, 'group2')
+
+      expect(described_class.ungrouped_queues).to eq(%w(queue1 queue2 queue2))
+    end
+  end
+
+  describe '.sqs_client_receive_message_opts' do
+    before do
+      Shoryuken.sqs_client_receive_message_opts
+    end
+
+    specify do
+      Shoryuken.sqs_client_receive_message_opts = { test: 1 }
+      expect(Shoryuken.sqs_client_receive_message_opts).to eq('default' => { test: 1 })
+
+      Shoryuken.sqs_client_receive_message_opts['group1'] = { test: 2 }
+
+      expect(Shoryuken.sqs_client_receive_message_opts).to eq(
+        'default' => { test: 1 },
+        'group1'  => { test: 2 },
+      )
+    end
+  end
+
+  describe '.register_worker' do
+    it 'registers a worker' do
+      described_class.worker_registry.clear
+      described_class.register_worker('default', TestWorker)
+      expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
+    end
+
+    it 'registers a batchable worker' do
+      described_class.worker_registry.clear
+      TestWorker.get_shoryuken_options['batch'] = true
+      described_class.register_worker('default', TestWorker)
+      expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
+    end
+
+    it 'allows multiple workers' do
+      described_class.worker_registry.clear
+      described_class.register_worker('default', TestWorker)
+      expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
+
+      class Test2Worker
+        include Shoryuken::Worker
+
+        shoryuken_options queue: 'default'
+
+        def perform(sqs_msg, body); end
+      end
+
+      expect(described_class.worker_registry.workers('default')).to eq([Test2Worker])
+    end
+
+    it 'raises an exception when mixing batchable with non batchable' do
+      described_class.worker_registry.clear
+      TestWorker.get_shoryuken_options['batch'] = true
+      described_class.register_worker('default', TestWorker)
+
+      expect {
+        class BatchableWorker
+          include Shoryuken::Worker
+
+          shoryuken_options queue: 'default', batch: true
+
+          def perform(sqs_msg, body); end
+        end
+      }.to raise_error("Could not register BatchableWorker for default, because TestWorker is already registered for this queue, " \
+        "and Shoryuken doesn't support a batchable worker for a queue with multiple workers")
+    end
+  end
+end

data/spec/shoryuken/{polling_spec.rb → polling/strict_priority_spec.rb}

@@ -1,105 +1,6 @@
 require 'spec_helper'
-require 'shoryuken/polling'
 
-describe Shoryuken::Polling::WeightedRoundRobin do
-  let(:queue1) { 'shoryuken' }
-  let(:queue2) { 'uppercut' }
-  let(:queues) { Array.new }
-  subject { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
-
-  describe '#next_queue' do
-    it 'cycles' do
-      # [shoryuken, 2]
-      # [uppercut,  1]
-      queues << queue1
-      queues << queue1
-      queues << queue2
-
-      expect(subject.next_queue).to eq(queue1)
-      expect(subject.next_queue).to eq(queue2)
-      expect(subject.next_queue).to eq(queue1)
-    end
-
-    it 'returns nil if there are no active queues' do
-      expect(subject.next_queue).to eq(nil)
-    end
-
-    it 'unpauses queues whose pause is expired' do
-      # [shoryuken, 2]
-      # [uppercut,  1]
-      queues << queue1
-      queues << queue1
-      queues << queue2
-
-      allow(subject).to receive(:delay).and_return(10)
-
-      now = Time.now
-      allow(Time).to receive(:now).and_return(now)
-
-      # pause the first queue
-      subject.messages_found(queue1, 0)
-      expect(subject.next_queue).to eq(queue2)
-
-      now += 5
-      allow(Time).to receive(:now).and_return(now)
-
-      # pause the second queue
-      subject.messages_found(queue2, 0)
-      expect(subject.next_queue).to eq(nil)
-
-      # queue1 should be unpaused now
-      now += 6
-      allow(Time).to receive(:now).and_return(now)
-      expect(subject.next_queue).to eq(queue1)
-
-      # queue1 should be unpaused and added to the end of queues now
-      now += 6
-      allow(Time).to receive(:now).and_return(now)
-      expect(subject.next_queue).to eq(queue1)
-      expect(subject.next_queue).to eq(queue2)
-    end
-  end
-
-  describe '#messages_found' do
-    it 'pauses a queue if there are no messages found' do
-      # [shoryuken, 2]
-      # [uppercut,  1]
-      queues << queue1
-      queues << queue1
-      queues << queue2
-
-      expect(subject).to receive(:pause).with(queue1).and_call_original
-      subject.messages_found(queue1, 0)
-      expect(subject.instance_variable_get(:@queues)).to eq([queue2])
-    end
-
-    it 'increased the weight if message is found' do
-      # [shoryuken, 2]
-      # [uppercut,  1]
-      queues << queue1
-      queues << queue1
-      queues << queue2
-
-      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2])
-      subject.messages_found(queue1, 1)
-      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
-    end
-
-    it 'respects the maximum queue weight' do
-      # [shoryuken, 2]
-      # [uppercut,  1]
-      queues << queue1
-      queues << queue1
-      queues << queue2
-
-      subject.messages_found(queue1, 1)
-      subject.messages_found(queue1, 1)
-      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
-    end
-  end
-end
-
-describe Shoryuken::Polling::StrictPriority do
+RSpec.describe Shoryuken::Polling::StrictPriority do
   let(:queue1) { 'shoryuken' }
   let(:queue2) { 'uppercut' }
   let(:queue3) { 'other' }

data/spec/shoryuken/polling/weighted_round_robin_spec.rb

@@ -0,0 +1,99 @@
+require 'spec_helper'
+
+RSpec.describe Shoryuken::Polling::WeightedRoundRobin do
+  let(:queue1) { 'shoryuken' }
+  let(:queue2) { 'uppercut' }
+  let(:queues) { Array.new }
+  subject { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
+
+  describe '#next_queue' do
+    it 'cycles' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      expect(subject.next_queue).to eq(queue1)
+    end
+
+    it 'returns nil if there are no active queues' do
+      expect(subject.next_queue).to eq(nil)
+    end
+
+    it 'unpauses queues whose pause is expired' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      allow(subject).to receive(:delay).and_return(10)
+
+      now = Time.now
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the first queue
+      subject.messages_found(queue1, 0)
+      expect(subject.next_queue).to eq(queue2)
+
+      now += 5
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the second queue
+      subject.messages_found(queue2, 0)
+      expect(subject.next_queue).to eq(nil)
+
+      # queue1 should be unpaused now
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue1)
+
+      # queue1 should be unpaused and added to the end of queues now
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+    end
+  end
+
+  describe '#messages_found' do
+    it 'pauses a queue if there are no messages found' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject).to receive(:pause).with(queue1).and_call_original
+      subject.messages_found(queue1, 0)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue2])
+    end
+
+    it 'increased the weight if message is found' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2])
+      subject.messages_found(queue1, 1)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
+    end
+
+    it 'respects the maximum queue weight' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      subject.messages_found(queue1, 1)
+      subject.messages_found(queue1, 1)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
+    end
+  end
+end

data/spec/shoryuken/processor_spec.rb

@@ -3,7 +3,7 @@ require 'shoryuken/processor'
 require 'shoryuken/manager'
 
 RSpec.describe Shoryuken::Processor do
-  let(:manager)   { double Shoryuken::Manager, processor_done: nil }
+  let(:manager)   { double Shoryuken::Manager }
   let(:sqs_queue) { double Shoryuken::Queue, visibility_timeout: 30 }
   let(:queue)     { 'default' }
 
@@ -16,14 +16,14 @@ RSpec.describe Shoryuken::Processor do
       receipt_handle: SecureRandom.uuid
   end
 
-  subject { described_class.new(manager) }
-
   before do
     allow(manager).to receive(:async).and_return(manager)
     allow(manager).to receive(:real_thread)
     allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
   end
 
+  subject { described_class.new(queue, sqs_msg) }
+
   describe '#process' do
     it 'parses the body into JSON' do
       TestWorker.get_shoryuken_options['body_parser'] = :json
@@ -34,7 +34,7 @@ RSpec.describe Shoryuken::Processor do
 
       allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'parses the body calling the proc' do
@@ -44,7 +44,7 @@ RSpec.describe Shoryuken::Processor do
 
       allow(sqs_msg).to receive(:body).and_return('test')
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'parses the body as text' do
@@ -56,7 +56,7 @@ RSpec.describe Shoryuken::Processor do
 
       allow(sqs_msg).to receive(:body).and_return(body)
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'parses calling `.load`' do
@@ -72,7 +72,7 @@ RSpec.describe Shoryuken::Processor do
 
       allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'parses calling `.parse`' do
@@ -88,28 +88,21 @@ RSpec.describe Shoryuken::Processor do
 
       allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     context 'when parse errors' do
       before do
         TestWorker.get_shoryuken_options['body_parser'] = :json
 
-        allow(sqs_msg).to receive(:body).and_return('invalid json')
+        allow(sqs_msg).to receive(:body).and_return('invalid JSON')
       end
 
-      it 'logs the error' do
-        expect(subject.logger).to receive(:error) do |&block|
-          expect(block.call).
-            to include("unexpected token at 'invalid json'\nbody_parser: json\nsqs_msg.body: invalid json")
-        end
-
-        subject.process(queue, sqs_msg) rescue nil
-      end
+      specify do
+        expect(subject.logger).to receive(:error).twice
 
-      it 're raises the error' do
-        expect { subject.process(queue, sqs_msg) }.
-          to raise_error(JSON::ParserError, /unexpected token at 'invalid json'/)
+        expect { subject.process }.
+          to raise_error(JSON::ParserError, /unexpected token at 'invalid JSON'/)
       end
     end
 
@@ -123,7 +116,7 @@ RSpec.describe Shoryuken::Processor do
 
         allow(sqs_msg).to receive(:body).and_return(body)
 
-        subject.process(queue, sqs_msg)
+        subject.process
       end
     end
 
@@ -150,7 +143,7 @@ RSpec.describe Shoryuken::Processor do
 
       context 'server' do
         before do
-          allow(Shoryuken).to receive(:server?).and_return(true)
+          allow(Shoryuken::Options).to receive(:server?).and_return(true)
           WorkerCalledMiddlewareWorker.instance_variable_set(:@server_chain, nil) # un-memoize middleware
 
           Shoryuken.configure_server do |config|
@@ -169,12 +162,10 @@ RSpec.describe Shoryuken::Processor do
         end
 
         it 'invokes middleware' do
-          expect(manager).to receive(:processor_done).with(queue)
-
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:called).with(sqs_msg, queue)
 
-          subject.process(queue, sqs_msg)
+          subject.process
         end
       end
 
@@ -199,12 +190,10 @@ RSpec.describe Shoryuken::Processor do
         end
 
         it "doesn't invoke middleware" do
-          expect(manager).to receive(:processor_done).with(queue)
-
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to_not receive(:called).with(sqs_msg, queue)
 
-          subject.process(queue, sqs_msg)
+          subject.process
         end
       end
     end
@@ -212,25 +201,21 @@ RSpec.describe Shoryuken::Processor do
     it 'performs with delete' do
       TestWorker.get_shoryuken_options['auto_delete'] = true
 
-      expect(manager).to receive(:processor_done).with(queue)
-
       expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
 
       expect(sqs_queue).to receive(:delete_messages).with(entries: [{ id: '0', receipt_handle: sqs_msg.receipt_handle }])
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'performs without delete' do
       TestWorker.get_shoryuken_options['auto_delete'] = false
 
-      expect(manager).to receive(:processor_done).with(queue)
-
       expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
 
       expect(sqs_queue).to_not receive(:delete_messages)
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     context 'when shoryuken_class header' do
@@ -249,13 +234,11 @@ RSpec.describe Shoryuken::Processor do
       it 'performs without delete' do
         Shoryuken.worker_registry.clear # unregister TestWorker
 
-        expect(manager).to receive(:processor_done).with(queue)
-
         expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
 
         expect(sqs_queue).to_not receive(:delete_messages)
 
-        subject.process(queue, sqs_msg)
+        subject.process
       end
     end
   end

data/spec/shoryuken/queue_spec.rb

@@ -1,22 +1,68 @@
 require 'spec_helper'
 
-describe Shoryuken::Queue do
+RSpec.describe Shoryuken::Queue do
   let(:credentials) { Aws::Credentials.new('access_key_id', 'secret_access_key') }
   let(:sqs) { Aws::SQS::Client.new(stub_responses: true, credentials: credentials) }
   let(:queue_name) { 'shoryuken' }
-  let(:queue_url) { 'https://eu-west-1.amazonaws.com:6059/123456789012/shoryuken' }
+  let(:queue_url) { "https://sqs.eu-west-1.amazonaws.com:6059/0123456789/#{queue_name}" }
 
   subject { described_class.new(sqs, queue_name) }
-  before {
+
+  before do
     # Required as Aws::SQS::Client.get_queue_url returns 'String' when responses are stubbed,
     # which is not accepted by Aws::SQS::Client.get_queue_attributes for :queue_name parameter.
     allow(subject).to receive(:url).and_return(queue_url)
-  }
+  end
+
+  describe '#new' do
+    context 'when queue url supplied' do
+      subject { described_class.new(sqs, queue_url) }
+
+      specify do
+        expect(subject.name).to eq(queue_name)
+      end
+    end
+
+    context 'when queue name supplied' do
+      subject { described_class.new(sqs, queue_name) }
+
+      specify do
+        expect(subject.name).to eq(queue_name)
+      end
+    end
+  end
+
+  describe '#delete_messages' do
+    let(:entries) do
+      [
+        { id: '1', receipt_handle:  '1' },
+        { id: '2', receipt_handle:  '2' }
+      ]
+    end
+
+    it 'deletes' do
+      expect(sqs).to receive(:delete_message_batch).with(entries: entries, queue_url: queue_url).and_return(double(failed: []))
+
+      subject.delete_messages(entries: entries)
+    end
+
+    context 'when it fails' do
+      it 'logs the reason' do
+        failure = double(id: 'id', code: 'code', message: '...', sender_fault: false)
+        logger = double 'Logger'
+
+        expect(sqs).to receive(:delete_message_batch).with(entries: entries, queue_url: queue_url).and_return(double(failed: [failure]))
+        expect(subject).to receive(:logger).and_return(logger)
+        expect(logger).to receive(:error)
+
+        subject.delete_messages(entries: entries)
+      end
+    end
+  end
 
   describe '#send_message' do
-    before {
-      allow(subject).to receive(:fifo?).and_return(false)
-    }
+    before { allow(subject).to receive(:fifo?).and_return(false) }
+
     it 'accepts SQS request parameters' do
       # https://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#send_message-instance_method
       expect(sqs).to receive(:send_message).with(hash_including(message_body: 'msg1'))
@@ -77,24 +123,24 @@ describe Shoryuken::Queue do
 
     it 'accepts an array of messages' do
       options = { entries: [{ id: '0',
-                              message_body: 'msg1',
-                              delay_seconds: 1,
-                              message_attributes: { attr: 'attr1' } },
-                            { id: '1',
-                              message_body: 'msg2',
-                              delay_seconds: 1,
-                              message_attributes: { attr: 'attr2' } }] }
+        message_body: 'msg1',
+        delay_seconds: 1,
+        message_attributes: { attr: 'attr1' } },
+        { id: '1',
+          message_body: 'msg2',
+          delay_seconds: 1,
+          message_attributes: { attr: 'attr2' } }] }
 
       expect(sqs).to receive(:send_message_batch).with(hash_including(options))
 
       subject.send_messages([{ message_body:       'msg1',
-                               delay_seconds:      1,
-                               message_attributes: { attr: 'attr1' }
-                             }, {
-                               message_body:       'msg2',
-                               delay_seconds:      1,
-                               message_attributes: { attr: 'attr2' }
-                             }])
+        delay_seconds:      1,
+        message_attributes: { attr: 'attr1' }
+      }, {
+        message_body:       'msg2',
+        delay_seconds:      1,
+        message_attributes: { attr: 'attr2' }
+      }])
     end
 
     context 'when FIFO' do
@@ -125,9 +171,9 @@ describe Shoryuken::Queue do
           end
 
           subject.send_messages([{ message_body: 'msg1',
-                                   message_attributes: { attr: 'attr1' },
-                                   message_group_id: 'my group',
-                                   message_deduplication_id: 'my id' }])
+            message_attributes: { attr: 'attr1' },
+            message_group_id: 'my group',
+            message_deduplication_id: 'my id' }])
         end
       end
     end
@@ -151,13 +197,13 @@ describe Shoryuken::Queue do
     context 'when queue is FIFO' do
       let(:fifo) { true }
 
-      it { expect(subject.fifo?).to be }
+      specify { expect(subject.fifo?).to be }
     end
 
     context 'when queue is not FIFO' do
       let(:fifo) { false }
 
-      it { expect(subject.fifo?).to_not be }
+      specify { expect(subject.fifo?).to_not be }
     end
   end
 end

data/spec/shoryuken/runner_spec.rb

@@ -1,6 +1,5 @@
 require 'spec_helper'
 require 'shoryuken/runner'
-require 'shoryuken/launcher'
 
 # rubocop:disable Metrics/BlockLength
 RSpec.describe Shoryuken::Runner do
@@ -17,10 +16,10 @@ RSpec.describe Shoryuken::Runner do
   describe '#run' do
     let(:launcher) { instance_double('Shoryuken::Launcher') }
 
-    before(:each) do
+    before do
       allow(Shoryuken::Launcher).to receive(:new).and_return(launcher)
-      allow(launcher).to receive(:run).and_raise(Interrupt)
-      allow(launcher).to receive(:stop)
+      allow(launcher).to receive(:start).and_raise(Interrupt)
+      allow(launcher).to receive(:stop!)
     end
 
     it 'does not raise' do