shoryuken 2.1.1 → 2.1.2

data/spec/shoryuken/middleware/server/auto_delete_spec.rb

@@ -55,8 +55,8 @@ describe Shoryuken::Middleware::Server::AutoDelete do
       expect(sqs_queue).to_not receive(:delete_messages)
 
       expect {
-        subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'Error' }
-      }.to raise_error(RuntimeError, 'Error')
+        subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' }
+      }.to raise_error('failed')
     end
   end
 end

data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb

@@ -46,7 +46,7 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
       allow(sqs_msg).to receive(:queue){ sqs_queue }
       expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 300)
 
-      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
+      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
     end
 
     it 'retries the job with exponential backoff' do
@@ -56,7 +56,7 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
       allow(sqs_msg).to receive(:queue){ sqs_queue }
       expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
 
-      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
+      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
     end
 
     it 'uses the last retry interval when :receive_count exceeds the size of :retry_intervals' do
@@ -66,7 +66,7 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
       allow(sqs_msg).to receive(:queue){ sqs_queue }
       expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
 
-      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
+      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
     end
 
     it 'limits the visibility timeout to 12 hours from receipt of message' do
@@ -75,7 +75,7 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
       allow(sqs_msg).to receive(:queue){ sqs_queue }
       expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 43198)
 
-      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
+      expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
     end
   end
 end

data/spec/shoryuken/polling_spec.rb

@@ -0,0 +1,239 @@
+require 'spec_helper'
+require 'shoryuken/polling'
+
+describe Shoryuken::Polling::WeightedRoundRobin do
+  let(:queue1) { 'shoryuken' }
+  let(:queue2) { 'uppercut' }
+  let(:queues) { Array.new }
+  subject { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
+
+  describe '#next_queue' do
+    it 'cycles' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      expect(subject.next_queue).to eq(queue1)
+    end
+
+    it 'returns nil if there are no active queues' do
+      expect(subject.next_queue).to eq(nil)
+    end
+
+    it 'unpauses queues whose pause is expired' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      allow(subject).to receive(:delay).and_return(10)
+
+      now = Time.now
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the first queue
+      subject.messages_found(queue1, 0)
+      expect(subject.next_queue).to eq(queue2)
+
+      now += 5
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the second queue
+      subject.messages_found(queue2, 0)
+      expect(subject.next_queue).to eq(nil)
+
+      # queue1 should be unpaused now
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue1)
+
+      # queue1 should be unpaused and added to the end of queues now
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+    end
+  end
+
+  describe '#messages_found' do
+    it 'pauses a queue if there are no messages found' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject).to receive(:pause).with(queue1).and_call_original
+      subject.messages_found(queue1, 0)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue2])
+    end
+
+    it 'increased the weight if message is found' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2])
+      subject.messages_found(queue1, 1)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
+    end
+
+    it 'respects the maximum queue weight' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      subject.messages_found(queue1, 1)
+      subject.messages_found(queue1, 1)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
+    end
+  end
+end
+
+describe Shoryuken::Polling::StrictPriority do
+  let(:queue1) { 'shoryuken' }
+  let(:queue2) { 'uppercut' }
+  let(:queue3) { 'other' }
+  let(:queues) { Array.new }
+  subject { Shoryuken::Polling::StrictPriority.new(queues) }
+
+  describe '#next_queue' do
+    it 'cycles when declared desc' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+    end
+
+    it 'cycles when declared asc' do
+      # [uppercut,  1]
+      # [shoryuken, 2]
+      queues << queue2
+      queues << queue1
+      queues << queue1
+
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+    end
+
+    it 'returns nil if there are no active queues' do
+      expect(subject.next_queue).to eq(nil)
+    end
+
+    it 'unpauses queues whose pause is expired' do
+      # [shoryuken, 3]
+      # [uppercut,  2]
+      # [other,     1]
+      queues << queue1
+      queues << queue1
+      queues << queue1
+      queues << queue2
+      queues << queue2
+      queues << queue3
+
+      allow(subject).to receive(:delay).and_return(10)
+
+      now = Time.now
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the second queue, see it loop between 1 and 3
+      subject.messages_found(queue2, 0)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue3)
+      expect(subject.next_queue).to eq(queue1)
+
+      now += 5
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the first queue, see it repeat 3
+      subject.messages_found(queue1, 0)
+      expect(subject.next_queue).to eq(queue3)
+      expect(subject.next_queue).to eq(queue3)
+
+      # pause the third queue, see it have nothing
+      subject.messages_found(queue3, 0)
+      expect(subject.next_queue).to eq(nil)
+
+      # unpause queue 2
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue2)
+
+      # unpause queues 1 and 3
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      expect(subject.next_queue).to eq(queue3)
+    end
+  end
+
+  describe '#messages_found' do
+    it 'pauses a queue if there are no messages found' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.active_queues).to eq([[queue1, 2], [queue2, 1]])
+      expect(subject).to receive(:pause).with(queue1).and_call_original
+      subject.messages_found(queue1, 0)
+      expect(subject.active_queues).to eq([[queue2, 1]])
+    end
+
+    it 'continues to queue the highest priority queue if messages are found' do
+      # [shoryuken, 3]
+      # [uppercut,  2]
+      # [other,     1]
+      queues << queue1
+      queues << queue1
+      queues << queue1
+      queues << queue2
+      queues << queue2
+      queues << queue3
+
+      expect(subject.next_queue).to eq(queue1)
+      subject.messages_found(queue1, 1)
+      expect(subject.next_queue).to eq(queue1)
+      subject.messages_found(queue1, 1)
+      expect(subject.next_queue).to eq(queue1)
+    end
+
+    it 'resets the priorities if messages are found part way' do
+      # [shoryuken, 3]
+      # [uppercut,  2]
+      # [other,     1]
+      queues << queue1
+      queues << queue1
+      queues << queue1
+      queues << queue2
+      queues << queue2
+      queues << queue3
+
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      subject.messages_found(queue2, 1)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      expect(subject.next_queue).to eq(queue3)
+    end
+  end
+end

data/test_workers/endless_interruptive_worker.rb

@@ -0,0 +1,41 @@
+class EndlessInterruptiveWorker
+  include Shoryuken::Worker
+
+  # Usage:
+  # QUEUE="super-q"
+  # MAX_EXECUTION_TIME=2000 QUEUE=$QUEUE \
+  # bundle exec ./bin/shoryuken -r ./examples/endless_uninterruptive_worker.rb -q $QUEUE -c 8
+
+  class << self
+    def queue
+      ENV['QUEUE'] || 'default'
+    end
+
+    def max_execution_time
+      ENV["MAX_EXECUTION_TIME"] ? ENV["MAX_EXECUTION_TIME"].to_i : 100
+    end
+
+    def rng
+      @rng ||= Random.new
+    end
+
+    # returns a random number between 0 and 100
+    def random_number(hi = 1000)
+      (rng.rand * hi).to_i
+    end
+  end
+
+  def perform(sqs_msg, body)
+    Shoryuken.logger.info("Received message: '#{body}'")
+
+    execution_ms = self.class.random_number(self.class.max_execution_time)
+    Shoryuken.logger.info("Going to sleep for #{execution_ms}ms")
+
+    new_body = "#{execution_ms}-" + body.to_s
+    sleep(execution_ms.to_f / 1000)
+
+    self.class.perform_async(new_body.slice(0, 512))
+  end
+
+  shoryuken_options queue: queue, auto_delete: true
+end

data/test_workers/endless_uninterruptive_worker.rb

@@ -0,0 +1,44 @@
+class EndlessUninterruptiveWorker
+  include Shoryuken::Worker
+
+  # Usage:
+  # QUEUE="super-q"
+  # MAX_EXECUTION_TIME=2000 QUEUE=$QUEUE \
+  # bundle exec ./bin/shoryuken -r ./examples/endless_interruptive_worker.rb -q $QUEUE -c 8
+
+  class << self
+    def queue
+      ENV['QUEUE'] || 'default'
+    end
+
+    def max_execution_time
+      ENV["MAX_EXECUTION_TIME"] ? ENV["MAX_EXECUTION_TIME"].to_i : 100
+    end
+
+    def rng
+      @rng ||= Random.new
+    end
+
+    # returns a random number between 0 and 100
+    def random_number(hi = 1000)
+      (rng.rand * hi).to_i
+    end
+  end
+
+  def perform(sqs_msg, body)
+    Shoryuken.logger.info("Received message: '#{body}'")
+
+    execution_ms = self.class.random_number(self.class.max_execution_time)
+    Shoryuken.logger.info("Going to burn metal for #{execution_ms}ms")
+    end_time = Time.now + execution_ms.to_f / 1000
+    while Time.now < end_time do
+      # burn metal
+    end
+
+    new_body = "#{execution_ms}-" + body.to_s
+
+    self.class.perform_async(new_body.slice(0, 512))
+  end
+
+  shoryuken_options queue: queue, auto_delete: true
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: shoryuken
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.1.2
 platform: ruby
 authors:
 - Pablo Cantero
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-05 00:00:00.000000000 Z
+date: 2016-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -164,6 +164,7 @@ files:
 - lib/shoryuken/middleware/server/auto_extend_visibility.rb
 - lib/shoryuken/middleware/server/exponential_backoff_retry.rb
 - lib/shoryuken/middleware/server/timing.rb
+- lib/shoryuken/polling.rb
 - lib/shoryuken/processor.rb
 - lib/shoryuken/queue.rb
 - lib/shoryuken/sns_arn.rb
@@ -187,6 +188,7 @@ files:
 - spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb
 - spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
 - spec/shoryuken/middleware/server/timing_spec.rb
+- spec/shoryuken/polling_spec.rb
 - spec/shoryuken/processor_spec.rb
 - spec/shoryuken/queue_spec.rb
 - spec/shoryuken/sns_arn_spec.rb
@@ -196,6 +198,8 @@ files:
 - spec/shoryuken_endpoint.yml
 - spec/shoryuken_spec.rb
 - spec/spec_helper.rb
+- test_workers/endless_interruptive_worker.rb
+- test_workers/endless_uninterruptive_worker.rb
 homepage: https://github.com/phstc/shoryuken
 licenses:
 - LGPL-3.0
@@ -234,6 +238,7 @@ test_files:
 - spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb
 - spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
 - spec/shoryuken/middleware/server/timing_spec.rb
+- spec/shoryuken/polling_spec.rb
 - spec/shoryuken/processor_spec.rb
 - spec/shoryuken/queue_spec.rb
 - spec/shoryuken/sns_arn_spec.rb