shoryuken 3.0.6 → 4.0.0

data/spec/shoryuken/options_spec.rb

@@ -0,0 +1,180 @@
+require 'spec_helper'
+
+RSpec.describe Shoryuken::Options do
+  describe '.add_group' do
+    before do
+      Shoryuken.groups.clear
+      Shoryuken.add_group('group1', 25)
+      Shoryuken.add_group('group2', 25)
+    end
+
+    specify do
+      described_class.add_queue('queue1', 1, 'group1')
+      described_class.add_queue('queue2', 2, 'group2')
+
+      expect(described_class.groups['group1'][:queues]).to eq(%w[queue1])
+      expect(described_class.groups['group2'][:queues]).to eq(%w[queue2 queue2])
+    end
+  end
+
+  describe '.ungrouped_queues' do
+    before do
+      Shoryuken.groups.clear
+      Shoryuken.add_group('group1', 25)
+      Shoryuken.add_group('group2', 25)
+    end
+
+    specify do
+      described_class.add_queue('queue1', 1, 'group1')
+      described_class.add_queue('queue2', 2, 'group2')
+
+      expect(described_class.ungrouped_queues).to eq(%w[queue1 queue2 queue2])
+    end
+  end
+
+  describe '.sqs_client_receive_message_opts' do
+    before do
+      Shoryuken.sqs_client_receive_message_opts
+    end
+
+    specify do
+      Shoryuken.sqs_client_receive_message_opts = { test: 1 }
+      expect(Shoryuken.sqs_client_receive_message_opts).to eq('default' => { test: 1 })
+
+      Shoryuken.sqs_client_receive_message_opts['group1'] = { test: 2 }
+
+      expect(Shoryuken.sqs_client_receive_message_opts).to eq(
+        'default' => { test: 1 },
+        'group1'  => { test: 2 }
+      )
+    end
+  end
+
+  describe '.register_worker' do
+    it 'registers a worker' do
+      described_class.worker_registry.clear
+      described_class.register_worker('default', TestWorker)
+      expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
+    end
+
+    it 'registers a batchable worker' do
+      described_class.worker_registry.clear
+      TestWorker.get_shoryuken_options['batch'] = true
+      described_class.register_worker('default', TestWorker)
+      expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
+    end
+
+    it 'allows multiple workers' do
+      described_class.worker_registry.clear
+      described_class.register_worker('default', TestWorker)
+      expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
+
+      class Test2Worker
+        include Shoryuken::Worker
+
+        shoryuken_options queue: 'default'
+
+        def perform(sqs_msg, body); end
+      end
+
+      expect(described_class.worker_registry.workers('default')).to eq([Test2Worker])
+    end
+
+    it 'raises an exception when mixing batchable with non batchable' do
+      described_class.worker_registry.clear
+      TestWorker.get_shoryuken_options['batch'] = true
+      described_class.register_worker('default', TestWorker)
+
+      expect {
+        class BatchableWorker
+          include Shoryuken::Worker
+
+          shoryuken_options queue: 'default', batch: true
+
+          def perform(sqs_msg, body); end
+        end
+      }.to raise_error('Could not register BatchableWorker for default, because TestWorker is already registered for this queue, ' \
+        "and Shoryuken doesn't support a batchable worker for a queue with multiple workers")
+    end
+  end
+
+  describe '.polling_strategy' do
+    context 'when not set' do
+      specify do
+        expect(Shoryuken.polling_strategy('default')).to eq Shoryuken::Polling::WeightedRoundRobin
+        expect(Shoryuken.polling_strategy('group1')).to eq Shoryuken::Polling::WeightedRoundRobin
+      end
+    end
+
+    context 'when set to StrictPriority string' do
+      before do
+        Shoryuken.options[:polling_strategy] = 'StrictPriority'
+
+        Shoryuken.options[:groups] = {
+          'group1' => {
+            polling_strategy: 'StrictPriority'
+          }
+        }
+      end
+
+      specify do
+        expect(Shoryuken.polling_strategy('default')).to eq Shoryuken::Polling::StrictPriority
+        expect(Shoryuken.polling_strategy('group1')).to eq Shoryuken::Polling::StrictPriority
+      end
+    end
+
+    context 'when set to WeightedRoundRobin string' do
+      before do
+        Shoryuken.options[:polling_strategy] = 'WeightedRoundRobin'
+
+        Shoryuken.options[:groups] = {
+          'group1' => {
+            polling_strategy: 'WeightedRoundRobin'
+          }
+        }
+      end
+
+      specify do
+        expect(Shoryuken.polling_strategy('default')).to eq Shoryuken::Polling::WeightedRoundRobin
+        expect(Shoryuken.polling_strategy('group1')).to eq Shoryuken::Polling::WeightedRoundRobin
+      end
+    end
+
+    context 'when set to non-existent string' do
+      before do
+        Shoryuken.options[:polling_strategy] = 'NonExistent1'
+
+        Shoryuken.options[:groups] = {
+          'group1' => {
+            polling_strategy: 'NonExistent2'
+          }
+        }
+      end
+
+      specify do
+        expect { Shoryuken.polling_strategy('default') }.to raise_error(ArgumentError)
+        expect { Shoryuken.polling_strategy('group1') }.to raise_error(ArgumentError)
+      end
+    end
+
+    context 'when set to a class' do
+      before do
+        class Foo < Shoryuken::Polling::BaseStrategy; end
+        class Bar < Shoryuken::Polling::BaseStrategy; end
+
+        Shoryuken.options[:polling_strategy] = Foo
+
+        Shoryuken.options[:groups] = {
+          'group1' => {
+            polling_strategy: Bar
+          }
+        }
+      end
+
+      specify do
+        expect(Shoryuken.polling_strategy('default')).to eq Foo
+        expect(Shoryuken.polling_strategy('group1')).to eq Bar
+      end
+    end
+  end
+end

data/spec/shoryuken/{polling_spec.rb → polling/strict_priority_spec.rb}

@@ -1,109 +1,10 @@
 require 'spec_helper'
-require 'shoryuken/polling'
 
-describe Shoryuken::Polling::WeightedRoundRobin do
-  let(:queue1) { 'shoryuken' }
-  let(:queue2) { 'uppercut' }
-  let(:queues) { Array.new }
-  subject { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
-
-  describe '#next_queue' do
-    it 'cycles' do
-      # [shoryuken, 2]
-      # [uppercut,  1]
-      queues << queue1
-      queues << queue1
-      queues << queue2
-
-      expect(subject.next_queue).to eq(queue1)
-      expect(subject.next_queue).to eq(queue2)
-      expect(subject.next_queue).to eq(queue1)
-    end
-
-    it 'returns nil if there are no active queues' do
-      expect(subject.next_queue).to eq(nil)
-    end
-
-    it 'unpauses queues whose pause is expired' do
-      # [shoryuken, 2]
-      # [uppercut,  1]
-      queues << queue1
-      queues << queue1
-      queues << queue2
-
-      allow(subject).to receive(:delay).and_return(10)
-
-      now = Time.now
-      allow(Time).to receive(:now).and_return(now)
-
-      # pause the first queue
-      subject.messages_found(queue1, 0)
-      expect(subject.next_queue).to eq(queue2)
-
-      now += 5
-      allow(Time).to receive(:now).and_return(now)
-
-      # pause the second queue
-      subject.messages_found(queue2, 0)
-      expect(subject.next_queue).to eq(nil)
-
-      # queue1 should be unpaused now
-      now += 6
-      allow(Time).to receive(:now).and_return(now)
-      expect(subject.next_queue).to eq(queue1)
-
-      # queue1 should be unpaused and added to the end of queues now
-      now += 6
-      allow(Time).to receive(:now).and_return(now)
-      expect(subject.next_queue).to eq(queue1)
-      expect(subject.next_queue).to eq(queue2)
-    end
-  end
-
-  describe '#messages_found' do
-    it 'pauses a queue if there are no messages found' do
-      # [shoryuken, 2]
-      # [uppercut,  1]
-      queues << queue1
-      queues << queue1
-      queues << queue2
-
-      expect(subject).to receive(:pause).with(queue1).and_call_original
-      subject.messages_found(queue1, 0)
-      expect(subject.instance_variable_get(:@queues)).to eq([queue2])
-    end
-
-    it 'increased the weight if message is found' do
-      # [shoryuken, 2]
-      # [uppercut,  1]
-      queues << queue1
-      queues << queue1
-      queues << queue2
-
-      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2])
-      subject.messages_found(queue1, 1)
-      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
-    end
-
-    it 'respects the maximum queue weight' do
-      # [shoryuken, 2]
-      # [uppercut,  1]
-      queues << queue1
-      queues << queue1
-      queues << queue2
-
-      subject.messages_found(queue1, 1)
-      subject.messages_found(queue1, 1)
-      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
-    end
-  end
-end
-
-describe Shoryuken::Polling::StrictPriority do
+RSpec.describe Shoryuken::Polling::StrictPriority do
   let(:queue1) { 'shoryuken' }
   let(:queue2) { 'uppercut' }
   let(:queue3) { 'other' }
-  let(:queues) { Array.new }
+  let(:queues) { [] }
   subject { Shoryuken::Polling::StrictPriority.new(queues) }
 
   describe '#next_queue' do

data/spec/shoryuken/polling/weighted_round_robin_spec.rb

@@ -0,0 +1,99 @@
+require 'spec_helper'
+
+RSpec.describe Shoryuken::Polling::WeightedRoundRobin do
+  let(:queue1) { 'shoryuken' }
+  let(:queue2) { 'uppercut' }
+  let(:queues) { [] }
+  subject { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
+
+  describe '#next_queue' do
+    it 'cycles' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      expect(subject.next_queue).to eq(queue1)
+    end
+
+    it 'returns nil if there are no active queues' do
+      expect(subject.next_queue).to eq(nil)
+    end
+
+    it 'unpauses queues whose pause is expired' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      allow(subject).to receive(:delay).and_return(10)
+
+      now = Time.now
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the first queue
+      subject.messages_found(queue1, 0)
+      expect(subject.next_queue).to eq(queue2)
+
+      now += 5
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the second queue
+      subject.messages_found(queue2, 0)
+      expect(subject.next_queue).to eq(nil)
+
+      # queue1 should be unpaused now
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue1)
+
+      # queue1 should be unpaused and added to the end of queues now
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+    end
+  end
+
+  describe '#messages_found' do
+    it 'pauses a queue if there are no messages found' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject).to receive(:pause).with(queue1).and_call_original
+      subject.messages_found(queue1, 0)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue2])
+    end
+
+    it 'increased the weight if message is found' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2])
+      subject.messages_found(queue1, 1)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
+    end
+
+    it 'respects the maximum queue weight' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      subject.messages_found(queue1, 1)
+      subject.messages_found(queue1, 1)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
+    end
+  end
+end

data/spec/shoryuken/processor_spec.rb

@@ -1,139 +1,44 @@
 require 'spec_helper'
-require 'shoryuken/processor'
-require 'shoryuken/manager'
 
 RSpec.describe Shoryuken::Processor do
-  let(:manager)   { double Shoryuken::Manager, processor_done: nil }
+  let(:manager)   { double Shoryuken::Manager }
   let(:sqs_queue) { double Shoryuken::Queue, visibility_timeout: 30 }
   let(:queue)     { 'default' }
 
   let(:sqs_msg) do
-    double Shoryuken::Message,
+    double(
+      Shoryuken::Message,
       queue_url: queue,
       body: 'test',
       message_attributes: {},
       message_id: SecureRandom.uuid,
       receipt_handle: SecureRandom.uuid
+    )
   end
 
-  subject { described_class.new(manager) }
-
   before do
     allow(manager).to receive(:async).and_return(manager)
     allow(manager).to receive(:real_thread)
     allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
   end
 
-  describe '#process' do
-    it 'parses the body into JSON' do
-      TestWorker.get_shoryuken_options['body_parser'] = :json
-
-      body = { 'test' => 'hi' }
-
-      expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, body)
-
-      allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
-
-      subject.process(queue, sqs_msg)
-    end
-
-    it 'parses the body calling the proc' do
-      TestWorker.get_shoryuken_options['body_parser'] = proc { |sqs_msg| "*#{sqs_msg.body}*" }
-
-      expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, '*test*')
-
-      allow(sqs_msg).to receive(:body).and_return('test')
-
-      subject.process(queue, sqs_msg)
-    end
-
-    it 'parses the body as text' do
-      TestWorker.get_shoryuken_options['body_parser'] = :text
-
-      body = 'test'
-
-      expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, body)
-
-      allow(sqs_msg).to receive(:body).and_return(body)
-
-      subject.process(queue, sqs_msg)
-    end
-
-    it 'parses calling `.load`' do
-      TestWorker.get_shoryuken_options['body_parser'] = Class.new do
-        def self.load(*args)
-          JSON.load(*args)
-        end
-      end
-
-      body = { 'test' => 'hi' }
-
-      expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, body)
-
-      allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
-
-      subject.process(queue, sqs_msg)
-    end
-
-    it 'parses calling `.parse`' do
-      TestWorker.get_shoryuken_options['body_parser'] = Class.new do
-        def self.parse(*args)
-          JSON.parse(*args)
-        end
-      end
-
-      body = { 'test' => 'hi' }
-
-      expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, body)
-
-      allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
-
-      subject.process(queue, sqs_msg)
-    end
+  subject { described_class.new(queue, sqs_msg) }
 
-    context 'when parse errors' do
-      before do
-        TestWorker.get_shoryuken_options['body_parser'] = :json
-
-        allow(sqs_msg).to receive(:body).and_return('invalid json')
-      end
-
-      it 'logs the error' do
-        expect(manager).to receive(:processor_failed)
-        expect(subject.logger).to receive(:error) do |&block|
-          expect(block.call).
-            to include("unexpected token at 'invalid json'\nbody_parser: json\nsqs_msg.body: invalid json")
-        end
-
-        subject.process(queue, sqs_msg) rescue nil
-      end
-
-      it 're raises the error' do
-        expect(manager).to receive(:processor_failed)
-        expect { subject.process(queue, sqs_msg) }.
-          to raise_error(JSON::ParserError, /unexpected token at 'invalid json'/)
-      end
-    end
-
-    context 'when `object_type: nil`' do
-      it 'parses the body as text' do
-        TestWorker.get_shoryuken_options['body_parser'] = nil
-
-        body = 'test'
-
-        expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, body)
+  describe '#process' do
+    it 'sets log context' do
+      expect(Shoryuken::Logging).to receive(:with_context).with("TestWorker/#{queue}/#{sqs_msg.message_id}")
 
-        allow(sqs_msg).to receive(:body).and_return(body)
+      allow_any_instance_of(TestWorker).to receive(:perform)
+      allow(sqs_msg).to receive(:body)
 
-        subject.process(queue, sqs_msg)
-      end
+      subject.process
     end
 
     context 'when custom middleware' do
       let(:queue) { 'worker_called_middleware' }
 
       class WorkerCalledMiddleware
-        def call(worker, queue, sqs_msg, body)
+        def call(worker, queue, sqs_msg, _body)
           # called is defined with `allow(...).to receive(...)`
           worker.called(sqs_msg, queue)
           yield
@@ -152,7 +57,7 @@ RSpec.describe Shoryuken::Processor do
 
       context 'server' do
         before do
-          allow(Shoryuken).to receive(:server?).and_return(true)
+          allow(Shoryuken::Options).to receive(:server?).and_return(true)
           WorkerCalledMiddlewareWorker.instance_variable_set(:@server_chain, nil) # un-memoize middleware
 
           Shoryuken.configure_server do |config|
@@ -171,12 +76,10 @@ RSpec.describe Shoryuken::Processor do
         end
 
         it 'invokes middleware' do
-          expect(manager).to receive(:processor_done).with(queue)
-
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:called).with(sqs_msg, queue)
 
-          subject.process(queue, sqs_msg)
+          subject.process
         end
       end
 
@@ -201,12 +104,10 @@ RSpec.describe Shoryuken::Processor do
         end
 
         it "doesn't invoke middleware" do
-          expect(manager).to receive(:processor_done).with(queue)
-
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to_not receive(:called).with(sqs_msg, queue)
 
-          subject.process(queue, sqs_msg)
+          subject.process
         end
       end
     end
@@ -214,50 +115,48 @@ RSpec.describe Shoryuken::Processor do
     it 'performs with delete' do
       TestWorker.get_shoryuken_options['auto_delete'] = true
 
-      expect(manager).to receive(:processor_done).with(queue)
-
       expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
 
       expect(sqs_queue).to receive(:delete_messages).with(entries: [{ id: '0', receipt_handle: sqs_msg.receipt_handle }])
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'performs without delete' do
       TestWorker.get_shoryuken_options['auto_delete'] = false
 
-      expect(manager).to receive(:processor_done).with(queue)
-
       expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
 
       expect(sqs_queue).to_not receive(:delete_messages)
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     context 'when shoryuken_class header' do
       let(:sqs_msg) do
-        double Shoryuken::Message,
+        double(
+          Shoryuken::Message,
           queue_url: queue,
           body: 'test',
           message_attributes: {
             'shoryuken_class' => {
               string_value: TestWorker.to_s,
-              data_type: 'String' }},
-              message_id: SecureRandom.uuid,
-              receipt_handle: SecureRandom.uuid
+              data_type: 'String'
+            }
+          },
+          message_id: SecureRandom.uuid,
+          receipt_handle: SecureRandom.uuid
+        )
       end
 
       it 'performs without delete' do
         Shoryuken.worker_registry.clear # unregister TestWorker
 
-        expect(manager).to receive(:processor_done).with(queue)
-
         expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
 
         expect(sqs_queue).to_not receive(:delete_messages)
 
-        subject.process(queue, sqs_msg)
+        subject.process
       end
     end
   end