shoryuken 2.1.1 → 3.1.3

data/spec/shoryuken/polling/weighted_round_robin_spec.rb

@@ -0,0 +1,99 @@
+require 'spec_helper'
+
+RSpec.describe Shoryuken::Polling::WeightedRoundRobin do
+  let(:queue1) { 'shoryuken' }
+  let(:queue2) { 'uppercut' }
+  let(:queues) { Array.new }
+  subject { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
+
+  describe '#next_queue' do
+    it 'cycles' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+      expect(subject.next_queue).to eq(queue1)
+    end
+
+    it 'returns nil if there are no active queues' do
+      expect(subject.next_queue).to eq(nil)
+    end
+
+    it 'unpauses queues whose pause is expired' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      allow(subject).to receive(:delay).and_return(10)
+
+      now = Time.now
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the first queue
+      subject.messages_found(queue1, 0)
+      expect(subject.next_queue).to eq(queue2)
+
+      now += 5
+      allow(Time).to receive(:now).and_return(now)
+
+      # pause the second queue
+      subject.messages_found(queue2, 0)
+      expect(subject.next_queue).to eq(nil)
+
+      # queue1 should be unpaused now
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue1)
+
+      # queue1 should be unpaused and added to the end of queues now
+      now += 6
+      allow(Time).to receive(:now).and_return(now)
+      expect(subject.next_queue).to eq(queue1)
+      expect(subject.next_queue).to eq(queue2)
+    end
+  end
+
+  describe '#messages_found' do
+    it 'pauses a queue if there are no messages found' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject).to receive(:pause).with(queue1).and_call_original
+      subject.messages_found(queue1, 0)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue2])
+    end
+
+    it 'increased the weight if message is found' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2])
+      subject.messages_found(queue1, 1)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
+    end
+
+    it 'respects the maximum queue weight' do
+      # [shoryuken, 2]
+      # [uppercut,  1]
+      queues << queue1
+      queues << queue1
+      queues << queue2
+
+      subject.messages_found(queue1, 1)
+      subject.messages_found(queue1, 1)
+      expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
+    end
+  end
+end

data/spec/shoryuken/processor_spec.rb

@@ -3,7 +3,7 @@ require 'shoryuken/processor'
 require 'shoryuken/manager'
 
 RSpec.describe Shoryuken::Processor do
-  let(:manager)   { double Shoryuken::Manager, processor_done: nil }
+  let(:manager)   { double Shoryuken::Manager }
   let(:sqs_queue) { double Shoryuken::Queue, visibility_timeout: 30 }
   let(:queue)     { 'default' }
 
@@ -16,14 +16,14 @@ RSpec.describe Shoryuken::Processor do
       receipt_handle: SecureRandom.uuid
   end
 
-  subject { described_class.new(manager) }
-
   before do
     allow(manager).to receive(:async).and_return(manager)
     allow(manager).to receive(:real_thread)
     allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
   end
 
+  subject { described_class.new(queue, sqs_msg) }
+
   describe '#process' do
     it 'parses the body into JSON' do
       TestWorker.get_shoryuken_options['body_parser'] = :json
@@ -34,7 +34,7 @@ RSpec.describe Shoryuken::Processor do
 
       allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'parses the body calling the proc' do
@@ -44,7 +44,7 @@ RSpec.describe Shoryuken::Processor do
 
       allow(sqs_msg).to receive(:body).and_return('test')
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'parses the body as text' do
@@ -56,7 +56,7 @@ RSpec.describe Shoryuken::Processor do
 
       allow(sqs_msg).to receive(:body).and_return(body)
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'parses calling `.load`' do
@@ -72,7 +72,7 @@ RSpec.describe Shoryuken::Processor do
 
       allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'parses calling `.parse`' do
@@ -88,28 +88,21 @@ RSpec.describe Shoryuken::Processor do
 
       allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     context 'when parse errors' do
       before do
         TestWorker.get_shoryuken_options['body_parser'] = :json
 
-        allow(sqs_msg).to receive(:body).and_return('invalid json')
+        allow(sqs_msg).to receive(:body).and_return('invalid JSON')
       end
 
-      it 'logs the error' do
-        expect(subject.logger).to receive(:error) do |&block|
-          expect(block.call).
-            to include("unexpected token at 'invalid json'\nbody_parser: json\nsqs_msg.body: invalid json")
-        end
-
-        subject.process(queue, sqs_msg) rescue nil
-      end
+      specify do
+        expect(subject.logger).to receive(:error).twice
 
-      it 're raises the error' do
-        expect { subject.process(queue, sqs_msg) }.
-          to raise_error(JSON::ParserError, /unexpected token at 'invalid json'/)
+        expect { subject.process }.
+          to raise_error(JSON::ParserError, /unexpected token at 'invalid JSON'/)
       end
     end
 
@@ -123,7 +116,7 @@ RSpec.describe Shoryuken::Processor do
 
         allow(sqs_msg).to receive(:body).and_return(body)
 
-        subject.process(queue, sqs_msg)
+        subject.process
       end
     end
 
@@ -150,7 +143,7 @@ RSpec.describe Shoryuken::Processor do
 
       context 'server' do
         before do
-          allow(Shoryuken).to receive(:server?).and_return(true)
+          allow(Shoryuken::Options).to receive(:server?).and_return(true)
           WorkerCalledMiddlewareWorker.instance_variable_set(:@server_chain, nil) # un-memoize middleware
 
           Shoryuken.configure_server do |config|
@@ -169,12 +162,10 @@ RSpec.describe Shoryuken::Processor do
         end
 
         it 'invokes middleware' do
-          expect(manager).to receive(:processor_done).with(queue, subject)
-
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:called).with(sqs_msg, queue)
 
-          subject.process(queue, sqs_msg)
+          subject.process
         end
       end
 
@@ -199,12 +190,10 @@ RSpec.describe Shoryuken::Processor do
         end
 
         it "doesn't invoke middleware" do
-          expect(manager).to receive(:processor_done).with(queue, subject)
-
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
           expect_any_instance_of(WorkerCalledMiddlewareWorker).to_not receive(:called).with(sqs_msg, queue)
 
-          subject.process(queue, sqs_msg)
+          subject.process
         end
       end
     end
@@ -212,25 +201,21 @@ RSpec.describe Shoryuken::Processor do
     it 'performs with delete' do
       TestWorker.get_shoryuken_options['auto_delete'] = true
 
-      expect(manager).to receive(:processor_done).with(queue, subject)
-
       expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
 
       expect(sqs_queue).to receive(:delete_messages).with(entries: [{ id: '0', receipt_handle: sqs_msg.receipt_handle }])
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     it 'performs without delete' do
       TestWorker.get_shoryuken_options['auto_delete'] = false
 
-      expect(manager).to receive(:processor_done).with(queue, subject)
-
       expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
 
       expect(sqs_queue).to_not receive(:delete_messages)
 
-      subject.process(queue, sqs_msg)
+      subject.process
     end
 
     context 'when shoryuken_class header' do
@@ -249,13 +234,11 @@ RSpec.describe Shoryuken::Processor do
       it 'performs without delete' do
         Shoryuken.worker_registry.clear # unregister TestWorker
 
-        expect(manager).to receive(:processor_done).with(queue, subject)
-
         expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
 
         expect(sqs_queue).to_not receive(:delete_messages)
 
-        subject.process(queue, sqs_msg)
+        subject.process
       end
     end
   end

data/spec/shoryuken/queue_spec.rb

@@ -1,22 +1,68 @@
 require 'spec_helper'
 
-describe Shoryuken::Queue do
+RSpec.describe Shoryuken::Queue do
   let(:credentials) { Aws::Credentials.new('access_key_id', 'secret_access_key') }
   let(:sqs) { Aws::SQS::Client.new(stub_responses: true, credentials: credentials) }
   let(:queue_name) { 'shoryuken' }
-  let(:queue_url) { 'https://eu-west-1.amazonaws.com:6059/123456789012/shoryuken' }
+  let(:queue_url) { "https://sqs.eu-west-1.amazonaws.com:6059/0123456789/#{queue_name}" }
 
   subject { described_class.new(sqs, queue_name) }
-  before {
+
+  before do
     # Required as Aws::SQS::Client.get_queue_url returns 'String' when responses are stubbed,
     # which is not accepted by Aws::SQS::Client.get_queue_attributes for :queue_name parameter.
     allow(subject).to receive(:url).and_return(queue_url)
-  }
+  end
+
+  describe '#new' do
+    context 'when queue url supplied' do
+      subject { described_class.new(sqs, queue_url) }
+
+      specify do
+        expect(subject.name).to eq(queue_name)
+      end
+    end
+
+    context 'when queue name supplied' do
+      subject { described_class.new(sqs, queue_name) }
+
+      specify do
+        expect(subject.name).to eq(queue_name)
+      end
+    end
+  end
+
+  describe '#delete_messages' do
+    let(:entries) do
+      [
+        { id: '1', receipt_handle:  '1' },
+        { id: '2', receipt_handle:  '2' }
+      ]
+    end
+
+    it 'deletes' do
+      expect(sqs).to receive(:delete_message_batch).with(entries: entries, queue_url: queue_url).and_return(double(failed: []))
+
+      subject.delete_messages(entries: entries)
+    end
+
+    context 'when it fails' do
+      it 'logs the reason' do
+        failure = double(id: 'id', code: 'code', message: '...', sender_fault: false)
+        logger = double 'Logger'
+
+        expect(sqs).to receive(:delete_message_batch).with(entries: entries, queue_url: queue_url).and_return(double(failed: [failure]))
+        expect(subject).to receive(:logger).and_return(logger)
+        expect(logger).to receive(:error)
+
+        subject.delete_messages(entries: entries)
+      end
+    end
+  end
 
   describe '#send_message' do
-    before {
-      allow(subject).to receive(:fifo?).and_return(false)
-    }
+    before { allow(subject).to receive(:fifo?).and_return(false) }
+
     it 'accepts SQS request parameters' do
       # https://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#send_message-instance_method
       expect(sqs).to receive(:send_message).with(hash_including(message_body: 'msg1'))
@@ -77,24 +123,24 @@ describe Shoryuken::Queue do
 
     it 'accepts an array of messages' do
       options = { entries: [{ id: '0',
-                              message_body: 'msg1',
-                              delay_seconds: 1,
-                              message_attributes: { attr: 'attr1' } },
-                            { id: '1',
-                              message_body: 'msg2',
-                              delay_seconds: 1,
-                              message_attributes: { attr: 'attr2' } }] }
+        message_body: 'msg1',
+        delay_seconds: 1,
+        message_attributes: { attr: 'attr1' } },
+        { id: '1',
+          message_body: 'msg2',
+          delay_seconds: 1,
+          message_attributes: { attr: 'attr2' } }] }
 
       expect(sqs).to receive(:send_message_batch).with(hash_including(options))
 
       subject.send_messages([{ message_body:       'msg1',
-                               delay_seconds:      1,
-                               message_attributes: { attr: 'attr1' }
-                             }, {
-                               message_body:       'msg2',
-                               delay_seconds:      1,
-                               message_attributes: { attr: 'attr2' }
-                             }])
+        delay_seconds:      1,
+        message_attributes: { attr: 'attr1' }
+      }, {
+        message_body:       'msg2',
+        delay_seconds:      1,
+        message_attributes: { attr: 'attr2' }
+      }])
     end
 
     context 'when FIFO' do
@@ -125,9 +171,9 @@ describe Shoryuken::Queue do
           end
 
           subject.send_messages([{ message_body: 'msg1',
-                                   message_attributes: { attr: 'attr1' },
-                                   message_group_id: 'my group',
-                                   message_deduplication_id: 'my id' }])
+            message_attributes: { attr: 'attr1' },
+            message_group_id: 'my group',
+            message_deduplication_id: 'my id' }])
         end
       end
     end
@@ -151,13 +197,13 @@ describe Shoryuken::Queue do
     context 'when queue is FIFO' do
       let(:fifo) { true }
 
-      it { expect(subject.fifo?).to be }
+      specify { expect(subject.fifo?).to be }
     end
 
     context 'when queue is not FIFO' do
       let(:fifo) { false }
 
-      it { expect(subject.fifo?).to_not be }
+      specify { expect(subject.fifo?).to_not be }
     end
   end
 end

data/spec/shoryuken/{cli_spec.rb → runner_spec.rb}

@@ -1,9 +1,9 @@
 require 'spec_helper'
-require 'shoryuken/cli'
-require 'shoryuken/launcher'
+require 'shoryuken/runner'
 
-RSpec.describe Shoryuken::CLI do
-  let(:cli) { Shoryuken::CLI.instance }
+# rubocop:disable Metrics/BlockLength
+RSpec.describe Shoryuken::Runner do
+  let(:cli) { Shoryuken::Runner.instance }
 
   before do
     # make sure we do not bail
@@ -16,50 +16,35 @@ RSpec.describe Shoryuken::CLI do
   describe '#run' do
     let(:launcher) { instance_double('Shoryuken::Launcher') }
 
-    before(:each) do
+    before do
       allow(Shoryuken::Launcher).to receive(:new).and_return(launcher)
-      allow(launcher).to receive(:run).and_raise(Interrupt)
-      allow(launcher).to receive(:stop)
+      allow(launcher).to receive(:start).and_raise(Interrupt)
+      allow(launcher).to receive(:stop!)
     end
 
     it 'does not raise' do
-      expect { cli.run([]) }.to_not raise_error
+      expect { cli.run({}) }.to_not raise_error
     end
 
     it 'daemonizes with --daemon --logfile' do
-      expect(cli).to receive(:celluloid_loaded?).and_return(false)
       expect(Process).to receive(:daemon)
-      cli.run(['--daemon', '--logfile', '/dev/null'])
+      cli.run(daemon: true, logfile: '/dev/null')
     end
 
     it 'does NOT daemonize with --logfile' do
       expect(Process).to_not receive(:daemon)
-      cli.run(['--logfile', '/dev/null'])
+      cli.run(logfile: '/dev/null')
     end
 
     it 'writes PID file with --pidfile' do
       pidfile = instance_double('File')
       expect(File).to receive(:open).with('/dev/null', 'w').and_yield(pidfile)
       expect(pidfile).to receive(:puts).with(Process.pid)
-      cli.run(['--pidfile', '/dev/null'])
+      cli.run(pidfile: '/dev/null')
     end
   end
 
   describe '#daemonize' do
-    before(:each) do
-      allow(cli).to receive(:celluloid_loaded?).and_return(false)
-    end
-
-    it 'raises if logfile is not set' do
-      expect { cli.send(:daemonize, daemon: true) }.to raise_error(ArgumentError)
-    end
-
-    it 'raises if Celluloid is already loaded' do
-      expect(cli).to receive(:celluloid_loaded?).and_return(true)
-      args = { daemon: true, logfile: '/dev/null' }
-      expect { cli.send(:daemonize, args) }.to raise_error(RuntimeError)
-    end
-
     it 'calls Process.daemon' do
       args = { daemon: true, logfile: '/dev/null' }
       expect(Process).to receive(:daemon).with(true, true)

data/spec/shoryuken_spec.rb

@@ -1,51 +1,4 @@
 require 'spec_helper'
 
-describe Shoryuken do
-  describe '.register_worker' do
-    it 'registers a worker' do
-      described_class.worker_registry.clear
-      described_class.register_worker('default', TestWorker)
-      expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
-    end
-
-    it 'registers a batchable worker' do
-      described_class.worker_registry.clear
-      TestWorker.get_shoryuken_options['batch'] = true
-      described_class.register_worker('default', TestWorker)
-      expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
-    end
-
-    it 'allows multiple workers' do
-      described_class.worker_registry.clear
-      described_class.register_worker('default', TestWorker)
-      expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
-
-      class Test2Worker
-        include Shoryuken::Worker
-
-        shoryuken_options queue: 'default'
-
-        def perform(sqs_msg, body); end
-      end
-
-      expect(described_class.worker_registry.workers('default')).to eq([Test2Worker])
-    end
-
-    it 'raises an exception when mixing batchable with non batchable' do
-      described_class.worker_registry.clear
-      TestWorker.get_shoryuken_options['batch'] = true
-      described_class.register_worker('default', TestWorker)
-
-      expect {
-        class BatchableWorker
-          include Shoryuken::Worker
-
-          shoryuken_options queue: 'default', batch: true
-
-          def perform(sqs_msg, body); end
-        end
-      }.to raise_error("Could not register BatchableWorker for 'default', because TestWorker is already registered for this queue, " \
-                       "and Shoryuken doesn't support a batchable worker for a queue with multiple workers")
-    end
-  end
+RSpec.describe Shoryuken do
 end

data/spec/spec_helper.rb

@@ -2,11 +2,10 @@ require 'bundler/setup'
 Bundler.setup
 
 require 'pry-byebug'
-require 'celluloid/current'
 require 'shoryuken'
 require 'json'
-require 'multi_xml'
 require 'dotenv'
+require 'securerandom'
 Dotenv.load
 
 if ENV['CODECLIMATE_REPO_TOKEN']
@@ -19,16 +18,6 @@ config_file = File.join(File.expand_path('../..', __FILE__), 'spec', 'shoryuken.
 Shoryuken::EnvironmentLoader.setup_options(config_file: config_file)
 
 Shoryuken.logger.level = Logger::UNKNOWN
-Celluloid.logger.level = Logger::UNKNOWN
-
-# I'm not sure whether this is an issue specific to running Shoryuken against github.com/comcast/cmb
-# as opposed to AWS itself, but sometimes the receive_messages call returns XML that looks like this:
-#
-# <ReceiveMessageResponse>\n\t<ReceiveMessageResult>\n\t</ReceiveMessageResult> ... </ReceiveMessageResponse>
-#
-# The default MultiXML parser is ReXML, which seems to mishandle \n\t chars. Nokogiri seems to be
-# the only one that correctly ignore this whitespace.
-MultiXml.parser = :nokogiri
 
 class TestWorker
   include Shoryuken::Worker
@@ -40,32 +29,37 @@ end
 
 RSpec.configure do |config|
   # Only run slow tests if SPEC_ALL=true and AWS_ACCESS_KEY_ID is present
-  # The AWS_ACCESS_KEY_ID checker is because Travis CI does not expose ENV variables to pull requests from forked repositories
+  # The AWS_ACCESS_KEY_ID checker is because Travis CI
+  # does not expose ENV variables to pull requests from forked repositories
   # http://docs.travis-ci.com/user/pull-requests/
-  config.filter_run_excluding slow: true if ENV['SPEC_ALL'] != 'true' || ENV['AWS_ACCESS_KEY_ID'].nil?
+  # config.filter_run_excluding slow: true if ENV['SPEC_ALL'] != 'true' || ENV['AWS_ACCESS_KEY_ID'].nil?
+  config.filter_run_excluding slow: true
 
   config.before do
     Shoryuken::Client.class_variable_set :@@queues, {}
-    Shoryuken::Client.class_variable_set :@@visibility_timeouts, {}
 
     Shoryuken::Client.sqs = nil
-    Shoryuken::Client.sqs_resource = nil
-    Shoryuken::Client.sns = nil
 
-    Shoryuken.queues.clear
+    Shoryuken.groups.clear
 
     Shoryuken.options[:concurrency] = 1
     Shoryuken.options[:delay]       = 1
     Shoryuken.options[:timeout]     = 1
     Shoryuken.options[:daemon]      = nil
     Shoryuken.options[:logfile]     = nil
-
-    Shoryuken.options[:aws].delete(:receive_message)
+    Shoryuken.options[:queues]      = nil
 
     TestWorker.get_shoryuken_options.clear
     TestWorker.get_shoryuken_options['queue'] = 'default'
 
+    Shoryuken.active_job_queue_name_prefixing = false
+
     Shoryuken.worker_registry.clear
     Shoryuken.register_worker('default', TestWorker)
+
+    Aws.config[:stub_responses] = true
+
+    allow(Concurrent).to receive(:global_io_executor).and_return(Concurrent::ImmediateExecutor.new)
+    allow(Shoryuken).to receive(:active_job?).and_return(false)
   end
 end

data/test_workers/endless_interruptive_worker.rb

@@ -0,0 +1,41 @@
+class EndlessInterruptiveWorker
+  include Shoryuken::Worker
+
+  # Usage:
+  # QUEUE="super-q"
+  # MAX_EXECUTION_TIME=2000 QUEUE=$QUEUE \
+  # bundle exec ./bin/shoryuken -r ./examples/endless_uninterruptive_worker.rb -q $QUEUE -c 8
+
+  class << self
+    def queue
+      ENV['QUEUE'] || 'default'
+    end
+
+    def max_execution_time
+      ENV["MAX_EXECUTION_TIME"] ? ENV["MAX_EXECUTION_TIME"].to_i : 100
+    end
+
+    def rng
+      @rng ||= Random.new
+    end
+
+    # returns a random number between 0 and 100
+    def random_number(hi = 1000)
+      (rng.rand * hi).to_i
+    end
+  end
+
+  def perform(sqs_msg, body)
+    Shoryuken.logger.info("Received message: '#{body}'")
+
+    execution_ms = self.class.random_number(self.class.max_execution_time)
+    Shoryuken.logger.info("Going to sleep for #{execution_ms}ms")
+
+    new_body = "#{execution_ms}-" + body.to_s
+    sleep(execution_ms.to_f / 1000)
+
+    self.class.perform_async(new_body.slice(0, 512))
+  end
+
+  shoryuken_options queue: queue, auto_delete: true
+end