shoryuken 3.0.11 → 3.1.0

data/lib/shoryuken/polling/base.rb

@@ -0,0 +1,67 @@
+module Shoryuken
+  module Polling
+    QueueConfiguration = Struct.new(:name, :options) do
+      def hash
+        name.hash
+      end
+
+      def ==(other)
+        case other
+        when String
+          if options.empty?
+            name == other
+          else
+            false
+          end
+        else
+          super
+        end
+      end
+
+      alias_method :eql?, :==
+
+      def to_s
+        if options.empty?
+          name
+        else
+          "#<QueueConfiguration #{name} options=#{options.inspect}>"
+        end
+      end
+    end
+
+    class BaseStrategy
+      include Util
+
+      def next_queue
+        fail NotImplementedError
+      end
+
+      def messages_found(queue, messages_found)
+        fail NotImplementedError
+      end
+
+      def active_queues
+        fail NotImplementedError
+      end
+
+      def ==(other)
+        case other
+        when Array
+          @queues == other
+        else
+          if other.respond_to?(:active_queues)
+            active_queues == other.active_queues
+          else
+            false
+          end
+        end
+      end
+
+      private
+
+      def delay
+        Shoryuken.options[:delay].to_f
+      end
+    end
+  end
+end

data/lib/shoryuken/polling/strict_priority.rb

@@ -0,0 +1,77 @@
+module Shoryuken
+  module Polling
+    class StrictPriority < BaseStrategy
+      def initialize(queues)
+        # Priority ordering of the queues, highest priority first
+        @queues = queues
+        .group_by { |q| q }
+        .sort_by { |_, qs| -qs.count }
+        .map(&:first)
+
+        # Pause status of the queues, default to past time (unpaused)
+        @paused_until = queues
+        .each_with_object(Hash.new) { |queue, h| h[queue] = Time.at(0) }
+
+        # Start queues at 0
+        reset_next_queue
+      end
+
+      def next_queue
+        next_queue = next_active_queue
+        next_queue.nil? ? nil : QueueConfiguration.new(next_queue, {})
+      end
+
+      def messages_found(queue, messages_found)
+        if messages_found == 0
+          pause(queue)
+        else
+          reset_next_queue
+        end
+      end
+
+      def active_queues
+        @queues
+        .reverse
+        .map.with_index(1)
+        .reject { |q, _| queue_paused?(q) }
+        .reverse
+      end
+
+      private
+
+      def next_active_queue
+        reset_next_queue if queues_unpaused_since?
+
+        size = @queues.length
+        size.times do
+          queue = @queues[@next_queue_index]
+          @next_queue_index = (@next_queue_index + 1) % size
+          return queue unless queue_paused?(queue)
+        end
+
+        nil
+      end
+
+      def queues_unpaused_since?
+        last = @last_unpause_check
+        now = @last_unpause_check = Time.now
+
+        last && @paused_until.values.any? { |t| t > last && t <= now }
+      end
+
+      def reset_next_queue
+        @next_queue_index = 0
+      end
+
+      def queue_paused?(queue)
+        @paused_until[queue] > Time.now
+      end
+
+      def pause(queue)
+        return unless delay > 0
+        @paused_until[queue] = Time.now + delay
+        logger.debug "Paused #{queue}"
+      end
+    end
+  end
+end

data/lib/shoryuken/polling/weighted_round_robin.rb

@@ -0,0 +1,66 @@
+module Shoryuken
+  module Polling
+    class WeightedRoundRobin < BaseStrategy
+      def initialize(queues)
+        @initial_queues = queues
+        @queues = queues.dup.uniq
+        @paused_queues = []
+      end
+
+      def next_queue
+        unpause_queues
+        queue = @queues.shift
+        return nil if queue.nil?
+
+        @queues << queue
+        QueueConfiguration.new(queue, {})
+      end
+
+      def messages_found(queue, messages_found)
+        if messages_found == 0
+          pause(queue)
+          return
+        end
+
+        maximum_weight = maximum_queue_weight(queue)
+        current_weight = current_queue_weight(queue)
+        if maximum_weight > current_weight
+          logger.info { "Increasing #{queue} weight to #{current_weight + 1}, max: #{maximum_weight}" }
+          @queues << queue
+        end
+      end
+
+      def active_queues
+        unparse_queues(@queues)
+      end
+
+      private
+
+      def pause(queue)
+        return unless @queues.delete(queue)
+        @paused_queues << [Time.now + delay, queue]
+        logger.debug "Paused #{queue}"
+      end
+
+      def unpause_queues
+        return if @paused_queues.empty?
+        return if Time.now < @paused_queues.first[0]
+        pause = @paused_queues.shift
+        @queues << pause[1]
+        logger.debug "Unpaused #{pause[1]}"
+      end
+
+      def current_queue_weight(queue)
+        queue_weight(@queues, queue)
+      end
+
+      def maximum_queue_weight(queue)
+        queue_weight(@initial_queues, queue)
+      end
+
+      def queue_weight(queues, queue)
+        queues.count { |q| q == queue }
+      end
+    end
+  end
+end

data/lib/shoryuken/processor.rb

@@ -2,35 +2,41 @@ module Shoryuken
   class Processor
     include Util
 
-    def initialize(manager)
-      @manager = manager
+    attr_reader :queue, :sqs_msg
+
+    def initialize(queue, sqs_msg)
+      @queue   = queue
+      @sqs_msg = sqs_msg
     end
 
-    def process(queue, sqs_msg)
-      worker = Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
-      body = get_body(worker.class, sqs_msg)
+    def process
+      return logger.error { "No worker found for #{queue}" } unless worker
 
       worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
         worker.perform(sqs_msg, body)
       end
     rescue Exception => ex
-      @manager.processor_failed(ex)
+      logger.error { "Processor failed: #{ex.message}" }
+      logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
+
       raise
-    ensure
-      @manager.processor_done(queue)
     end
 
     private
 
-    def get_body(worker_class, sqs_msg)
-      if sqs_msg.is_a? Array
-        sqs_msg.map { |m| parse_body(worker_class, m) }
-      else
-        parse_body(worker_class, sqs_msg)
-      end
+    def worker
+      @_worker ||= Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
     end
 
-    def parse_body(worker_class, sqs_msg)
+    def worker_class
+      worker.class
+    end
+
+    def body
+      @_body ||= sqs_msg.is_a?(Array) ? sqs_msg.map(&method(:parse_body)) : parse_body(sqs_msg)
+    end
+
+    def parse_body(sqs_msg)
       body_parser = worker_class.get_shoryuken_options['body_parser']
 
       case body_parser
@@ -50,9 +56,6 @@ module Shoryuken
           body_parser.load(sqs_msg.body)
         end
       end
-    rescue => ex
-      logger.error { "Error parsing the message body: #{ex.message}\nbody_parser: #{body_parser}\nsqs_msg.body: #{sqs_msg.body}" }
-      raise
     end
   end
 end

data/lib/shoryuken/runner.rb

@@ -19,7 +19,7 @@ module Shoryuken
     def run(options)
       self_read, self_write = IO.pipe
 
-      %w(INT TERM USR1 USR2 TTIN).each do |sig|
+      %w(INT TERM USR1 TTIN).each do |sig|
         begin
           trap sig do
             self_write.puts(sig)
@@ -43,22 +43,15 @@ module Shoryuken
 
       @launcher = Shoryuken::Launcher.new
 
-      if (callback = Shoryuken.start_callback)
-        logger.info { 'Calling on_start callback' }
-        callback.call
-      end
-
-      fire_event(:startup)
-
       begin
-        @launcher.run
+        @launcher.start
 
         while (readable_io = IO.select([self_read]))
           signal = readable_io.first[0].gets.strip
           handle_signal(signal)
         end
       rescue Interrupt
-        @launcher.stop(shutdown: true)
+        @launcher.stop!
         exit 0
       end
     end
@@ -110,7 +103,6 @@ module Shoryuken
       logger.info { 'Received USR1, will soft shutdown down' }
 
       @launcher.stop
-      fire_event(:quiet, true)
       exit 0
     end
 
@@ -126,13 +118,9 @@ module Shoryuken
     end
 
     def handle_signal(sig)
-      logger.info { "Got #{sig} signal" }
-
       case sig
       when 'USR1' then execute_soft_shutdown
       when 'TTIN' then print_threads_backtrace
-      when 'USR2'
-        logger.warn { "Received #{sig}, will do nothing. To execute soft shutdown, please send USR1" }
       else
         logger.info { "Received #{sig}, will shutdown down" }
 

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '3.0.11'.freeze
+  VERSION = '3.1.0'.freeze
 end

data/spec/integration/launcher_spec.rb

@@ -6,25 +6,31 @@ require 'securerandom'
 RSpec.describe Shoryuken::Launcher do
   describe 'Consuming messages', slow: :true do
     before do
+      Aws.config[:stub_responses] = false
+      Aws.config[:region] = 'us-east-1'
+
       StandardWorker.received_messages = 0
 
-      queue = "test_shoryuken#{StandardWorker}_#{SecureRandom.uuid}"
+      queue = "shoryuken-travis-#{StandardWorker}-#{SecureRandom.uuid}"
 
-      Shoryuken::Client.sqs.create_queue queue_name: queue
+      Shoryuken::Client.sqs.create_queue(queue_name: queue)
 
-      Shoryuken.queues << queue
+      Shoryuken.add_group('default', 1)
+      Shoryuken.add_queue(queue, 1, 'default')
 
       StandardWorker.get_shoryuken_options['queue'] = queue
 
-      Shoryuken.register_worker queue, StandardWorker
+      Shoryuken.register_worker(queue, StandardWorker)
     end
 
     after do
+      Aws.config[:stub_responses] = true
+
       queue_url = Shoryuken::Client.sqs.get_queue_url(
         queue_name: StandardWorker.get_shoryuken_options['queue']
       ).queue_url
 
-      Shoryuken::Client.sqs.delete_queue queue_url: queue_url
+      Shoryuken::Client.sqs.delete_queue(queue_url: queue_url)
     end
 
     it 'consumes as a command worker' do
@@ -61,7 +67,7 @@ RSpec.describe Shoryuken::Launcher do
     end
 
     def poll_queues_until
-      subject.run
+      subject.start
 
       Timeout::timeout(10) do
         begin

data/spec/shoryuken/environment_loader_spec.rb

@@ -15,20 +15,11 @@ RSpec.describe Shoryuken::EnvironmentLoader do
       allow(subject).to receive(:patch_deprecated_workers)
     end
 
-    it 'parses' do
-      Shoryuken.options[:queues] = ['queue_1']
+    specify do
+      Shoryuken.options[:queues] = ['queue1', ['queue2', 2]]
       subject.load
 
-      expect(Shoryuken.queues).to eq(%w(queue_1))
-    end
-
-    context 'with priority' do
-      it 'parses' do
-        Shoryuken.options[:queues] = ['queue_1', ['queue_2', 2]]
-        subject.load
-
-        expect(Shoryuken.queues).to eq(%w(queue_1 queue_2 queue_2))
-      end
+      expect(Shoryuken.groups['default'][:queues]).to eq(%w(queue1 queue2 queue2))
     end
   end
 end

data/spec/shoryuken/fetcher_spec.rb

@@ -2,36 +2,51 @@ require 'spec_helper'
 require 'shoryuken/manager'
 require 'shoryuken/fetcher'
 
-describe Shoryuken::Fetcher do
-  let(:queue)      { instance_double('Shoryuken::Queue') }
-  let(:queue_name) { 'default' }
+RSpec.describe Shoryuken::Fetcher do
+  let(:queue)        { instance_double('Shoryuken::Queue') }
+  let(:queue_name)   { 'default' }
   let(:queue_config) { Shoryuken::Polling::QueueConfiguration.new(queue_name, {}) }
+  let(:group)        { 'default' }
 
   let(:sqs_msg) do
-    double(Shoryuken::Message,
+    double(
+      Shoryuken::Message,
       queue_url: queue_name,
       body: 'test',
       message_id: 'fc754df79cc24c4196ca5996a44b771e',
-          )
+    )
   end
 
-  subject { described_class.new }
+  subject { described_class.new(group) }
 
   describe '#fetch' do
-    it 'calls Shoryuken::Client to receive messages' do
+    let(:limit) { 1 }
+
+    specify do
       expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
+
+      Shoryuken.sqs_client_receive_message_opts[group] = { wait_time_seconds: 10 }
+
       expect(queue).to receive(:receive_messages).
-        with(max_number_of_messages: 1, attribute_names: ['All'], message_attribute_names: ['All']).
+        with(wait_time_seconds: 10, max_number_of_messages: limit, message_attribute_names: ['All'], attribute_names: ['All']).
         and_return([])
-      subject.fetch(queue_config, 1)
+
+      subject.fetch(queue_config, limit)
     end
 
-    it 'maxes messages to receive to 10 (SQS limit)' do
-      allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
-      expect(queue).to receive(:receive_messages).
-        with(max_number_of_messages: 10, attribute_names: ['All'], message_attribute_names: ['All']).
-        and_return([])
-      subject.fetch(queue_config, 20)
+    context 'when limit is greater than FETCH_LIMIT' do
+      let(:limit) { 20 }
+
+      specify do
+        Shoryuken.sqs_client_receive_message_opts[group] = {}
+
+        allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
+        expect(queue).to receive(:receive_messages).
+          with(max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']).
+          and_return([])
+
+        subject.fetch(queue_config, limit)
+      end
     end
   end
 end