shoryuken 3.0.9 → 3.1.0

data/spec/shoryuken/polling/weighted_round_robin_spec.rb

@@ -0,0 +1,99 @@
+require 'spec_helper'
+
+RSpec.describe Shoryuken::Polling::WeightedRoundRobin do
+  let(:queue1) { 'shoryuken' }
+  let(:queue2) { 'uppercut' }
+  let(:queues) { Array.new }
+  subject { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
+
+  describe '#next_queue' do
+    it 'cycles' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      expect(subject.next_queue).to eq(queue1)
+    end
+
+    it 'returns nil if there are no active queues' do
+      expect(subject.next_queue).to eq(nil)
+    end
+
+    it 'unpauses queues whose pause is expired' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      allow(subject).to receive(:delay).and_return(10)
+
+      now = Time.now
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the first queue
+      subject.messages_found(queue1, 0)
+      expect(subject.next_queue).to eq(queue2)
+
+      now += 5
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the second queue
+      subject.messages_found(queue2, 0)
+      expect(subject.next_queue).to eq(nil)
+
+      # queue1 should be unpaused now
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue1)
+
+      # queue1 should be unpaused and added to the end of queues now
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+    end
+  end
+
+  describe '#messages_found' do
+    it 'pauses a queue if there are no messages found' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject).to receive(:pause).with(queue1).and_call_original
+      subject.messages_found(queue1, 0)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue2])
+    end
+
+    it 'increased the weight if message is found' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2])
+      subject.messages_found(queue1, 1)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
+    end
+
+    it 'respects the maximum queue weight' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      subject.messages_found(queue1, 1)
+      subject.messages_found(queue1, 1)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
+    end
+  end
+end

data/spec/shoryuken/processor_spec.rb

@@ -3,7 +3,7 @@ require 'shoryuken/processor'
 require 'shoryuken/manager'
 
 RSpec.describe Shoryuken::Processor do
-  let(:manager)   { double Shoryuken::Manager, processor_done: nil }
+  let(:manager)   { double Shoryuken::Manager }
   let(:sqs_queue) { double Shoryuken::Queue, visibility_timeout: 30 }
   let(:queue)     { 'default' }
 
@@ -16,14 +16,14 @@ RSpec.describe Shoryuken::Processor do
       receipt_handle: SecureRandom.uuid
   end
 
-  subject { described_class.new(manager) }
-
   before do
     allow(manager).to receive(:async).and_return(manager)
     allow(manager).to receive(:real_thread)
     allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
   end
 
+  subject { described_class.new(queue, sqs_msg) }
+
   describe '#process' do
     it 'parses the body into JSON' do
       TestWorker.get_shoryuken_options['body_parser'] = :json
@@ -34,7 +34,7 @@ RSpec.describe Shoryuken::Processor do
 
       allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'parses the body calling the proc' do
@@ -44,7 +44,7 @@ RSpec.describe Shoryuken::Processor do
 
       allow(sqs_msg).to receive(:body).and_return('test')
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'parses the body as text' do
@@ -56,7 +56,7 @@ RSpec.describe Shoryuken::Processor do
 
       allow(sqs_msg).to receive(:body).and_return(body)
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'parses calling `.load`' do
@@ -72,7 +72,7 @@ RSpec.describe Shoryuken::Processor do
 
       allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'parses calling `.parse`' do
@@ -88,30 +88,21 @@ RSpec.describe Shoryuken::Processor do
 
       allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     context 'when parse errors' do
       before do
         TestWorker.get_shoryuken_options['body_parser'] = :json
 
-        allow(sqs_msg).to receive(:body).and_return('invalid json')
+        allow(sqs_msg).to receive(:body).and_return('invalid JSON')
       end
 
-      it 'logs the error' do
-        expect(manager).to receive(:processor_failed)
-        expect(subject.logger).to receive(:error) do |&block|
-          expect(block.call).
-            to include("unexpected token at 'invalid json'\nbody_parser: json\nsqs_msg.body: invalid json")
-        end
-
-        subject.process(queue, sqs_msg) rescue nil
-      end
+      specify do
+        expect(subject.logger).to receive(:error).twice
 
-      it 're raises the error' do
-        expect(manager).to receive(:processor_failed)
-        expect { subject.process(queue, sqs_msg) }.
-          to raise_error(JSON::ParserError, /unexpected token at 'invalid json'/)
+        expect { subject.process }.
+          to raise_error(JSON::ParserError, /unexpected token at 'invalid JSON'/)
       end
     end
 
@@ -125,7 +116,7 @@ RSpec.describe Shoryuken::Processor do
 
         allow(sqs_msg).to receive(:body).and_return(body)
 
-        subject.process(queue, sqs_msg)
+        subject.process
       end
     end
 
@@ -152,7 +143,7 @@ RSpec.describe Shoryuken::Processor do
 
       context 'server' do
         before do
-          allow(Shoryuken).to receive(:server?).and_return(true)
+          allow(Shoryuken::Options).to receive(:server?).and_return(true)
           WorkerCalledMiddlewareWorker.instance_variable_set(:@server_chain, nil) # un-memoize middleware
 
           Shoryuken.configure_server do |config|
@@ -171,12 +162,10 @@ RSpec.describe Shoryuken::Processor do
         end
 
         it 'invokes middleware' do
-          expect(manager).to receive(:processor_done).with(queue)
-
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:called).with(sqs_msg, queue)
 
-          subject.process(queue, sqs_msg)
+          subject.process
         end
       end
 
@@ -201,12 +190,10 @@ RSpec.describe Shoryuken::Processor do
         end
 
         it "doesn't invoke middleware" do
-          expect(manager).to receive(:processor_done).with(queue)
-
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to_not receive(:called).with(sqs_msg, queue)
 
-          subject.process(queue, sqs_msg)
+          subject.process
         end
       end
     end
@@ -214,25 +201,21 @@ RSpec.describe Shoryuken::Processor do
     it 'performs with delete' do
       TestWorker.get_shoryuken_options['auto_delete'] = true
 
-      expect(manager).to receive(:processor_done).with(queue)
-
       expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
 
       expect(sqs_queue).to receive(:delete_messages).with(entries: [{ id: '0', receipt_handle: sqs_msg.receipt_handle }])
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'performs without delete' do
       TestWorker.get_shoryuken_options['auto_delete'] = false
 
-      expect(manager).to receive(:processor_done).with(queue)
-
       expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
 
       expect(sqs_queue).to_not receive(:delete_messages)
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     context 'when shoryuken_class header' do
@@ -251,13 +234,11 @@ RSpec.describe Shoryuken::Processor do
       it 'performs without delete' do
         Shoryuken.worker_registry.clear # unregister TestWorker
 
-        expect(manager).to receive(:processor_done).with(queue)
-
         expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
 
         expect(sqs_queue).to_not receive(:delete_messages)
 
-        subject.process(queue, sqs_msg)
+        subject.process
       end
     end
   end

data/spec/shoryuken/queue_spec.rb

@@ -22,6 +22,14 @@ RSpec.describe Shoryuken::Queue do
         expect(subject.name).to eq(queue_name)
       end
     end
+
+    context 'when queue name supplied' do
+      subject { described_class.new(sqs, queue_name) }
+
+      specify do
+        expect(subject.name).to eq(queue_name)
+      end
+    end
   end
 
   describe '#delete_messages' do

data/spec/shoryuken/runner_spec.rb

@@ -1,6 +1,5 @@
 require 'spec_helper'
 require 'shoryuken/runner'
-require 'shoryuken/launcher'
 
 # rubocop:disable Metrics/BlockLength
 RSpec.describe Shoryuken::Runner do
@@ -17,10 +16,10 @@ RSpec.describe Shoryuken::Runner do
   describe '#run' do
     let(:launcher) { instance_double('Shoryuken::Launcher') }
 
-    before(:each) do
+    before do
       allow(Shoryuken::Launcher).to receive(:new).and_return(launcher)
-      allow(launcher).to receive(:run).and_raise(Interrupt)
-      allow(launcher).to receive(:stop)
+      allow(launcher).to receive(:start).and_raise(Interrupt)
+      allow(launcher).to receive(:stop!)
     end
 
     it 'does not raise' do

data/spec/shoryuken_spec.rb

@@ -1,63 +1,4 @@
 require 'spec_helper'
 
 RSpec.describe Shoryuken do
-  describe '.add_queue' do
-    after { Shoryuken.queues.clear }
-
-    it 'adds queues' do
-      described_class.add_queue('default')
-      expect(described_class.queues).to eq(['default'])
-
-      described_class.add_queue('high', 2)
-      expect(described_class.queues).to eq(%w(default high high))
-    end
-  end
-
-  describe '.register_worker' do
-    it 'registers a worker' do
-      described_class.worker_registry.clear
-      described_class.register_worker('default', TestWorker)
-      expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
-    end
-
-    it 'registers a batchable worker' do
-      described_class.worker_registry.clear
-      TestWorker.get_shoryuken_options['batch'] = true
-      described_class.register_worker('default', TestWorker)
-      expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
-    end
-
-    it 'allows multiple workers' do
-      described_class.worker_registry.clear
-      described_class.register_worker('default', TestWorker)
-      expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
-
-      class Test2Worker
-        include Shoryuken::Worker
-
-        shoryuken_options queue: 'default'
-
-        def perform(sqs_msg, body); end
-      end
-
-      expect(described_class.worker_registry.workers('default')).to eq([Test2Worker])
-    end
-
-    it 'raises an exception when mixing batchable with non batchable' do
-      described_class.worker_registry.clear
-      TestWorker.get_shoryuken_options['batch'] = true
-      described_class.register_worker('default', TestWorker)
-
-      expect {
-        class BatchableWorker
-          include Shoryuken::Worker
-
-          shoryuken_options queue: 'default', batch: true
-
-          def perform(sqs_msg, body); end
-        end
-      }.to raise_error("Could not register BatchableWorker for 'default', because TestWorker is already registered for this queue, " \
-                       "and Shoryuken doesn't support a batchable worker for a queue with multiple workers")
-    end
-  end
 end

data/spec/spec_helper.rb

@@ -32,14 +32,15 @@ RSpec.configure do |config|
   # The AWS_ACCESS_KEY_ID checker is because Travis CI
   # does not expose ENV variables to pull requests from forked repositories
   # http://docs.travis-ci.com/user/pull-requests/
-  config.filter_run_excluding slow: true if ENV['SPEC_ALL'] != 'true' || ENV['AWS_ACCESS_KEY_ID'].nil?
+  # config.filter_run_excluding slow: true if ENV['SPEC_ALL'] != 'true' || ENV['AWS_ACCESS_KEY_ID'].nil?
+  config.filter_run_excluding slow: true
 
   config.before do
     Shoryuken::Client.class_variable_set :@@queues, {}
 
     Shoryuken::Client.sqs = nil
 
-    Shoryuken.queues.clear
+    Shoryuken.groups.clear
 
     Shoryuken.options[:concurrency] = 1
     Shoryuken.options[:delay]       = 1
@@ -53,5 +54,9 @@ RSpec.configure do |config|
 
     Shoryuken.worker_registry.clear
     Shoryuken.register_worker('default', TestWorker)
+
+    Aws.config[:stub_responses] = true
+
+    allow(Concurrent).to receive(:global_io_executor).and_return(Concurrent::ImmediateExecutor.new)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 3.0.9
+  version: 3.1.0
 platform: ruby
 authors:
 - Pablo Cantero
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-05 00:00:00.000000000 Z
+date: 2017-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -84,14 +84,14 @@ dependencies:
   name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
         version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
         version: '2'
 - !ruby/object:Gem::Dependency
@@ -162,7 +162,10 @@ files:
 - lib/shoryuken/middleware/server/auto_extend_visibility.rb
 - lib/shoryuken/middleware/server/exponential_backoff_retry.rb
 - lib/shoryuken/middleware/server/timing.rb
-- lib/shoryuken/polling.rb
+- lib/shoryuken/options.rb
+- lib/shoryuken/polling/base.rb
+- lib/shoryuken/polling/strict_priority.rb
+- lib/shoryuken/polling/weighted_round_robin.rb
 - lib/shoryuken/processor.rb
 - lib/shoryuken/queue.rb
 - lib/shoryuken/runner.rb
@@ -185,7 +188,9 @@ files:
 - spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb
 - spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
 - spec/shoryuken/middleware/server/timing_spec.rb
-- spec/shoryuken/polling_spec.rb
+- spec/shoryuken/options_spec.rb
+- spec/shoryuken/polling/strict_priority_spec.rb
+- spec/shoryuken/polling/weighted_round_robin_spec.rb
 - spec/shoryuken/processor_spec.rb
 - spec/shoryuken/queue_spec.rb
 - spec/shoryuken/runner_spec.rb
@@ -233,7 +238,9 @@ test_files:
 - spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb
 - spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
 - spec/shoryuken/middleware/server/timing_spec.rb
-- spec/shoryuken/polling_spec.rb
+- spec/shoryuken/options_spec.rb
+- spec/shoryuken/polling/strict_priority_spec.rb
+- spec/shoryuken/polling/weighted_round_robin_spec.rb
 - spec/shoryuken/processor_spec.rb
 - spec/shoryuken/queue_spec.rb
 - spec/shoryuken/runner_spec.rb

data/lib/shoryuken/polling.rb

@@ -1,204 +0,0 @@
-module Shoryuken
-  module Polling
-    QueueConfiguration = Struct.new(:name, :options) do
-      def hash
-        name.hash
-      end
-
-      def ==(other)
-        case other
-        when String
-          if options.empty?
-            name == other
-          else
-            false
-          end
-        else
-          super
-        end
-      end
-
-      alias_method :eql?, :==
-
-      def to_s
-        if options.empty?
-          name
-        else
-          "#<QueueConfiguration #{name} options=#{options.inspect}>"
-        end
-      end
-    end
-
-    class BaseStrategy
-      include Util
-
-      def next_queue
-        fail NotImplementedError
-      end
-
-      def messages_found(queue, messages_found)
-        fail NotImplementedError
-      end
-
-      def active_queues
-        fail NotImplementedError
-      end
-
-      def ==(other)
-        case other
-        when Array
-          @queues == other
-        else
-          if other.respond_to?(:active_queues)
-            active_queues == other.active_queues
-          else
-            false
-          end
-        end
-      end
-
-      private
-
-      def delay
-        Shoryuken.options[:delay].to_f
-      end
-    end
-
-    class WeightedRoundRobin < BaseStrategy
-      def initialize(queues)
-        @initial_queues = queues
-        @queues = queues.dup.uniq
-        @paused_queues = []
-      end
-
-      def next_queue
-        unpause_queues
-        queue = @queues.shift
-        return nil if queue.nil?
-
-        @queues << queue
-        QueueConfiguration.new(queue, {})
-      end
-
-      def messages_found(queue, messages_found)
-        if messages_found == 0
-          pause(queue)
-          return
-        end
-
-        maximum_weight = maximum_queue_weight(queue)
-        current_weight = current_queue_weight(queue)
-        if maximum_weight > current_weight
-          logger.info { "Increasing #{queue} weight to #{current_weight + 1}, max: #{maximum_weight}" }
-          @queues << queue
-        end
-      end
-
-      def active_queues
-        unparse_queues(@queues)
-      end
-
-      private
-
-      def pause(queue)
-        return unless @queues.delete(queue)
-        @paused_queues << [Time.now + delay, queue]
-        logger.debug "Paused #{queue}"
-      end
-
-      def unpause_queues
-        return if @paused_queues.empty?
-        return if Time.now < @paused_queues.first[0]
-        pause = @paused_queues.shift
-        @queues << pause[1]
-        logger.debug "Unpaused #{pause[1]}"
-      end
-
-      def current_queue_weight(queue)
-        queue_weight(@queues, queue)
-      end
-
-      def maximum_queue_weight(queue)
-        queue_weight(@initial_queues, queue)
-      end
-
-      def queue_weight(queues, queue)
-        queues.count { |q| q == queue }
-      end
-    end
-
-    class StrictPriority < BaseStrategy
-      def initialize(queues)
-        # Priority ordering of the queues, highest priority first
-        @queues = queues
-          .group_by { |q| q }
-          .sort_by { |_, qs| -qs.count }
-          .map(&:first)
-
-        # Pause status of the queues, default to past time (unpaused)
-        @paused_until = queues
-          .each_with_object(Hash.new) { |queue, h| h[queue] = Time.at(0) }
-
-        # Start queues at 0
-        reset_next_queue
-      end
-
-      def next_queue
-        next_queue = next_active_queue
-        next_queue.nil? ? nil : QueueConfiguration.new(next_queue, {})
-      end
-
-      def messages_found(queue, messages_found)
-        if messages_found == 0
-          pause(queue)
-        else
-          reset_next_queue
-        end
-      end
-
-      def active_queues
-        @queues
-          .reverse
-          .map.with_index(1)
-          .reject { |q, _| queue_paused?(q) }
-          .reverse
-      end
-
-      private
-
-      def next_active_queue
-        reset_next_queue if queues_unpaused_since?
-
-        size = @queues.length
-        size.times do
-          queue = @queues[@next_queue_index]
-          @next_queue_index = (@next_queue_index + 1) % size
-          return queue unless queue_paused?(queue)
-        end
-
-        nil
-      end
-
-      def queues_unpaused_since?
-        last = @last_unpause_check
-        now = @last_unpause_check = Time.now
-
-        last && @paused_until.values.any? { |t| t > last && t <= now }
-      end
-
-      def reset_next_queue
-        @next_queue_index = 0
-      end
-
-      def queue_paused?(queue)
-        @paused_until[queue] > Time.now
-      end
-
-      def pause(queue)
-        return unless delay > 0
-        @paused_until[queue] = Time.now + delay
-        logger.debug "Paused #{queue}"
-      end
-    end
-  end
-end