shoryuken 3.0.4 → 3.1.0

data/lib/shoryuken/processor.rb

@@ -2,32 +2,41 @@ module Shoryuken
   class Processor
     include Util
 
-    def initialize(manager)
-      @manager = manager
+    attr_reader :queue, :sqs_msg
+
+    def initialize(queue, sqs_msg)
+      @queue   = queue
+      @sqs_msg = sqs_msg
     end
 
-    def process(queue, sqs_msg)
-      worker = Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
-      body = get_body(worker.class, sqs_msg)
+    def process
+      return logger.error { "No worker found for #{queue}" } unless worker
 
       worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
         worker.perform(sqs_msg, body)
       end
-    ensure
-      @manager.processor_done(queue)
+    rescue Exception => ex
+      logger.error { "Processor failed: #{ex.message}" }
+      logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
+
+      raise
     end
 
     private
 
-    def get_body(worker_class, sqs_msg)
-      if sqs_msg.is_a? Array
-        sqs_msg.map { |m| parse_body(worker_class, m) }
-      else
-        parse_body(worker_class, sqs_msg)
-      end
+    def worker
+      @_worker ||= Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
     end
 
-    def parse_body(worker_class, sqs_msg)
+    def worker_class
+      worker.class
+    end
+
+    def body
+      @_body ||= sqs_msg.is_a?(Array) ? sqs_msg.map(&method(:parse_body)) : parse_body(sqs_msg)
+    end
+
+    def parse_body(sqs_msg)
       body_parser = worker_class.get_shoryuken_options['body_parser']
 
       case body_parser
@@ -47,9 +56,6 @@ module Shoryuken
           body_parser.load(sqs_msg.body)
         end
       end
-    rescue => e
-      logger.error { "Error parsing the message body: #{e.message}\nbody_parser: #{body_parser}\nsqs_msg.body: #{sqs_msg.body}" }
-      raise
     end
   end
 end

data/lib/shoryuken/queue.rb

@@ -1,17 +1,16 @@
 module Shoryuken
   class Queue
+    include Util
+
     FIFO_ATTR               = 'FifoQueue'
     MESSAGE_GROUP_ID        = 'ShoryukenMessage'
     VISIBILITY_TIMEOUT_ATTR = 'VisibilityTimeout'
 
     attr_accessor :name, :client, :url
 
-    def initialize(client, name)
-      self.name   = name
+    def initialize(client, name_or_url)
       self.client = client
-      self.url    = client.get_queue_url(queue_name: name).queue_url
-    rescue Aws::SQS::Errors::NonExistentQueue => e
-      raise e, "The specified queue '#{name}' does not exist."
+      set_name_and_url(name_or_url)
     end
 
     def visibility_timeout
@@ -19,7 +18,13 @@ module Shoryuken
     end
 
     def delete_messages(options)
-      client.delete_message_batch(options.merge(queue_url: url))
+      client.delete_message_batch(
+        options.merge(queue_url: url)
+      ).failed.any? do |failure|
+        logger.error do
+          "Could not delete #{failure.id}, code: '#{failure.code}', message: '#{failure.message}', sender_fault: #{failure.sender_fault}"
+        end
+      end
     end
 
     def send_message(options)
@@ -44,6 +49,22 @@ module Shoryuken
 
     private
 
+    def set_name(name)
+      self.name = name
+      self.url  = client.get_queue_url(queue_name: name).queue_url
+    rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue => ex
+      raise ex, "The specified queue #{name} does not exist."
+    end
+
+    def set_url(url)
+      self.name = url.split('/').last
+      self.url  = url
+    end
+
+    def set_name_and_url(name_or_url)
+      name_or_url.start_with?('https://sqs.') ? set_url(name_or_url) : set_name(name_or_url)
+    end
+
     def queue_attributes
       # Note: Retrieving all queue attributes as requesting `FifoQueue` on non-FIFO queue raises error.
       # See issue: https://github.com/aws/aws-sdk-ruby/issues/1350

data/lib/shoryuken/runner.rb

@@ -19,7 +19,7 @@ module Shoryuken
     def run(options)
       self_read, self_write = IO.pipe
 
-      %w(INT TERM USR1 USR2 TTIN).each do |sig|
+      %w(INT TERM USR1 TTIN).each do |sig|
         begin
           trap sig do
             self_write.puts(sig)
@@ -43,22 +43,15 @@ module Shoryuken
 
       @launcher = Shoryuken::Launcher.new
 
-      if (callback = Shoryuken.start_callback)
-        logger.info { 'Calling Shoryuken.on_start block' }
-        callback.call
-      end
-
-      fire_event(:startup)
-
       begin
-        @launcher.run
+        @launcher.start
 
         while (readable_io = IO.select([self_read]))
           signal = readable_io.first[0].gets.strip
           handle_signal(signal)
         end
       rescue Interrupt
-        @launcher.stop(shutdown: true)
+        @launcher.stop!
         exit 0
       end
     end
@@ -110,7 +103,6 @@ module Shoryuken
       logger.info { 'Received USR1, will soft shutdown down' }
 
       @launcher.stop
-      fire_event(:quiet, true)
       exit 0
     end
 
@@ -126,13 +118,9 @@ module Shoryuken
     end
 
     def handle_signal(sig)
-      logger.info { "Got #{sig} signal" }
-
       case sig
       when 'USR1' then execute_soft_shutdown
       when 'TTIN' then print_threads_backtrace
-      when 'USR2'
-        logger.warn { "Received #{sig}, will do nothing. To execute soft shutdown, please send USR1" }
       else
         logger.info { "Received #{sig}, will shutdown down" }
 

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '3.0.4'.freeze
+  VERSION = '3.1.0'.freeze
 end

data/lib/shoryuken/worker.rb

@@ -48,6 +48,14 @@ module Shoryuken
         !!get_shoryuken_options['auto_visibility_timeout']
       end
 
+      def exponential_backoff?
+        !!get_shoryuken_options['retry_intervals']
+      end
+
+      def auto_delete?
+        !!(get_shoryuken_options['delete'] || get_shoryuken_options['auto_delete'])
+      end
+
       def get_shoryuken_options # :nodoc:
         @shoryuken_options || Shoryuken.default_worker_options
       end

data/lib/shoryuken.rb

@@ -20,184 +20,51 @@ require 'shoryuken/middleware/server/auto_delete'
 Shoryuken::Middleware::Server.autoload :AutoExtendVisibility, 'shoryuken/middleware/server/auto_extend_visibility'
 require 'shoryuken/middleware/server/exponential_backoff_retry'
 require 'shoryuken/middleware/server/timing'
-require 'shoryuken/polling'
+require 'shoryuken/polling/base'
+require 'shoryuken/polling/weighted_round_robin'
+require 'shoryuken/polling/strict_priority'
 require 'shoryuken/manager'
 require 'shoryuken/launcher'
 require 'shoryuken/processor'
 require 'shoryuken/fetcher'
+require 'shoryuken/options'
 
 module Shoryuken
-  DEFAULTS = {
-    concurrency: 25,
-    queues: [],
-    aws: {},
-    delay: 0,
-    timeout: 8,
-    lifecycle_events: {
-      startup: [],
-      quiet: [],
-      shutdown: []
-    },
-    polling_strategy: Polling::WeightedRoundRobin
-  }.freeze
-
-  @@queues                          = []
-  @@worker_registry                 = DefaultWorkerRegistry.new
-  @@active_job_queue_name_prefixing = false
-  @@sqs_client                      = nil
-  @@sqs_client_receive_message_opts = {}
-  @@start_callback                  = nil
-  @@stop_callback                   = nil
-
-  class << self
-    def queues
-      @@queues
-    end
-
-    def add_queue(queue, priority = 1)
-      priority.times { queues << queue }
-    end
-
-    def worker_registry
-      @@worker_registry
-    end
-
-    def worker_registry=(worker_registry)
-      @@worker_registry = worker_registry
-    end
-
-    def start_callback
-      @@start_callback
-    end
-
-    def start_callback=(start_callback)
-      @@start_callback = start_callback
-    end
-
-    def stop_callback
-      @@stop_callback
-    end
-
-    def stop_callback=(stop_callback)
-      @@stop_callback = stop_callback
-    end
-
-    def active_job_queue_name_prefixing
-      @@active_job_queue_name_prefixing
-    end
-
-    def active_job_queue_name_prefixing=(active_job_queue_name_prefixing)
-      @@active_job_queue_name_prefixing = active_job_queue_name_prefixing
-    end
-
-    def sqs_client
-      @@sqs_client ||= Aws::SQS::Client.new
-    end
-
-    def sqs_client=(sqs_client)
-      @@sqs_client = sqs_client
-    end
-
-    def sqs_client_receive_message_opts
-      @@sqs_client_receive_message_opts
-    end
-
-    def sqs_client_receive_message_opts=(sqs_client_receive_message_opts)
-      @@sqs_client_receive_message_opts = sqs_client_receive_message_opts
-    end
-
-    def options
-      @@options ||= DEFAULTS.dup
-    end
-
-    def logger
-      Shoryuken::Logging.logger
-    end
-
-    def register_worker(*args)
-      @@worker_registry.register_worker(*args)
-    end
-
-    def configure_server
-      yield self if server?
-    end
-
-    def server_middleware
-      @@server_chain ||= default_server_middleware
-      yield @@server_chain if block_given?
-      @@server_chain
-    end
-
-    def configure_client
-      yield self unless server?
-    end
-
-    def client_middleware
-      @@client_chain ||= default_client_middleware
-      yield @@client_chain if block_given?
-      @@client_chain
-    end
-
-    def default_worker_options
-      @@default_worker_options ||= {
-        'queue'                   => 'default',
-        'delete'                  => false,
-        'auto_delete'             => false,
-        'auto_visibility_timeout' => false,
-        'retry_intervals'         => nil,
-        'batch'                   => false
-      }
-    end
-
-    def default_worker_options=(default_worker_options)
-      @@default_worker_options = default_worker_options
-    end
-
-    def on_start(&block)
-      @@start_callback = block
-    end
-
-    def on_stop(&block)
-      @@stop_callback = block
-    end
-
-    # Register a block to run at a point in the Shoryuken lifecycle.
-    # :startup, :quiet or :shutdown are valid events.
-    #
-    #   Shoryuken.configure_server do |config|
-    #     config.on(:shutdown) do
-    #       puts "Goodbye cruel world!"
-    #     end
-    #   end
-    def on(event, &block)
-      fail ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
-      fail ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
-      options[:lifecycle_events][event] << block
-    end
-
-    private
-
-    def default_server_middleware
-      Middleware::Chain.new do |m|
-        m.add Middleware::Server::Timing
-        m.add Middleware::Server::ExponentialBackoffRetry
-        m.add Middleware::Server::AutoDelete
-        m.add Middleware::Server::AutoExtendVisibility
-        if defined?(::ActiveRecord::Base)
-          require 'shoryuken/middleware/server/active_record'
-          m.add Middleware::Server::ActiveRecord
-        end
-      end
-    end
-
-    def default_client_middleware
-      Middleware::Chain.new
-    end
-
-    def server?
-      defined?(Shoryuken::CLI)
-    end
-  end
+  extend SingleForwardable
+
+  def_delegators(
+    :'Shoryuken::Options',
+    :add_group,
+    :groups,
+    :add_queue,
+    :ungrouped_queues,
+    :worker_registry,
+    :worker_registry=,
+    :polling_strategy,
+    :start_callback,
+    :start_callback=,
+    :stop_callback,
+    :stop_callback=,
+    :active_job_queue_name_prefixing,
+    :active_job_queue_name_prefixing=,
+    :sqs_client,
+    :sqs_client=,
+    :sqs_client_receive_message_opts,
+    :sqs_client_receive_message_opts=,
+    :options,
+    :logger,
+    :register_worker,
+    :configure_server,
+    :server?,
+    :server_middleware,
+    :configure_client,
+    :client_middleware,
+    :default_worker_options,
+    :default_worker_options=,
+    :on_start,
+    :on_stop,
+    :on
+  )
 end
 
 require 'shoryuken/extensions/active_job_adapter' if defined?(::ActiveJob)

data/shoryuken.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'pry-byebug'
   spec.add_development_dependency 'dotenv'
 
-  spec.add_dependency 'aws-sdk-core', '~> 2'
+  spec.add_dependency 'aws-sdk-core', '> 2'
   spec.add_dependency 'concurrent-ruby'
   spec.add_dependency 'thor'
 end

data/spec/integration/launcher_spec.rb

@@ -6,25 +6,31 @@ require 'securerandom'
 RSpec.describe Shoryuken::Launcher do
   describe 'Consuming messages', slow: :true do
     before do
+      Aws.config[:stub_responses] = false
+      Aws.config[:region] = 'us-east-1'
+
       StandardWorker.received_messages = 0
 
-      queue = "test_shoryuken#{StandardWorker}_#{SecureRandom.uuid}"
+      queue = "shoryuken-travis-#{StandardWorker}-#{SecureRandom.uuid}"
 
-      Shoryuken::Client.sqs.create_queue queue_name: queue
+      Shoryuken::Client.sqs.create_queue(queue_name: queue)
 
-      Shoryuken.queues << queue
+      Shoryuken.add_group('default', 1)
+      Shoryuken.add_queue(queue, 1, 'default')
 
       StandardWorker.get_shoryuken_options['queue'] = queue
 
-      Shoryuken.register_worker queue, StandardWorker
+      Shoryuken.register_worker(queue, StandardWorker)
     end
 
     after do
+      Aws.config[:stub_responses] = true
+
       queue_url = Shoryuken::Client.sqs.get_queue_url(
         queue_name: StandardWorker.get_shoryuken_options['queue']
       ).queue_url
 
-      Shoryuken::Client.sqs.delete_queue queue_url: queue_url
+      Shoryuken::Client.sqs.delete_queue(queue_url: queue_url)
     end
 
     it 'consumes as a command worker' do
@@ -61,7 +67,7 @@ RSpec.describe Shoryuken::Launcher do
     end
 
     def poll_queues_until
-      subject.run
+      subject.start
 
       Timeout::timeout(10) do
         begin

data/spec/shoryuken/environment_loader_spec.rb

@@ -15,20 +15,11 @@ RSpec.describe Shoryuken::EnvironmentLoader do
       allow(subject).to receive(:patch_deprecated_workers)
     end
 
-    it 'parses' do
-      Shoryuken.options[:queues] = ['queue_1']
+    specify do
+      Shoryuken.options[:queues] = ['queue1', ['queue2', 2]]
       subject.load
 
-      expect(Shoryuken.queues).to eq(%w(queue_1))
-    end
-
-    context 'with priority' do
-      it 'parses' do
-        Shoryuken.options[:queues] = ['queue_1', ['queue_2', 2]]
-        subject.load
-
-        expect(Shoryuken.queues).to eq(%w(queue_1 queue_2 queue_2))
-      end
+      expect(Shoryuken.groups['default'][:queues]).to eq(%w(queue1 queue2 queue2))
     end
   end
 end

data/spec/shoryuken/fetcher_spec.rb

@@ -2,36 +2,51 @@ require 'spec_helper'
 require 'shoryuken/manager'
 require 'shoryuken/fetcher'
 
-describe Shoryuken::Fetcher do
-  let(:queue)      { instance_double('Shoryuken::Queue') }
-  let(:queue_name) { 'default' }
+RSpec.describe Shoryuken::Fetcher do
+  let(:queue)        { instance_double('Shoryuken::Queue') }
+  let(:queue_name)   { 'default' }
   let(:queue_config) { Shoryuken::Polling::QueueConfiguration.new(queue_name, {}) }
+  let(:group)        { 'default' }
 
   let(:sqs_msg) do
-    double(Shoryuken::Message,
+    double(
+      Shoryuken::Message,
       queue_url: queue_name,
       body: 'test',
       message_id: 'fc754df79cc24c4196ca5996a44b771e',
-          )
+    )
   end
 
-  subject { described_class.new }
+  subject { described_class.new(group) }
 
   describe '#fetch' do
-    it 'calls Shoryuken::Client to receive messages' do
+    let(:limit) { 1 }
+
+    specify do
       expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
+
+      Shoryuken.sqs_client_receive_message_opts[group] = { wait_time_seconds: 10 }
+
       expect(queue).to receive(:receive_messages).
-        with(max_number_of_messages: 1, attribute_names: ['All'], message_attribute_names: ['All']).
+        with(wait_time_seconds: 10, max_number_of_messages: limit, message_attribute_names: ['All'], attribute_names: ['All']).
         and_return([])
-      subject.fetch(queue_config, 1)
+
+      subject.fetch(queue_config, limit)
     end
 
-    it 'maxes messages to receive to 10 (SQS limit)' do
-      allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
-      expect(queue).to receive(:receive_messages).
-        with(max_number_of_messages: 10, attribute_names: ['All'], message_attribute_names: ['All']).
-        and_return([])
-      subject.fetch(queue_config, 20)
+    context 'when limit is greater than FETCH_LIMIT' do
+      let(:limit) { 20 }
+
+      specify do
+        Shoryuken.sqs_client_receive_message_opts[group] = {}
+
+        allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
+        expect(queue).to receive(:receive_messages).
+          with(max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']).
+          and_return([])
+
+        subject.fetch(queue_config, limit)
+      end
     end
   end
 end

data/spec/shoryuken/manager_spec.rb

@@ -11,28 +11,20 @@ RSpec.describe Shoryuken::Manager do
   let(:queue) { 'default' }
   let(:queues) { [queue] }
   let(:polling_strategy) { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
-  let(:fetcher) { Shoryuken::Fetcher.new }
+  let(:fetcher) { double Shoryuken::Fetcher }
   let(:concurrency) { 1 }
 
-  subject { Shoryuken::Manager.new(fetcher, polling_strategy) }
+  subject { Shoryuken::Manager.new(fetcher, polling_strategy, concurrency) }
 
-  before(:each) do
-    Shoryuken.options[:concurrency] = concurrency
+  before do
+    allow(fetcher).to receive(:fetch).and_return([])
   end
 
-  after(:each) do
+  after do
     Shoryuken.options[:concurrency] = 1
     TestWorker.get_shoryuken_options['batch'] = false
   end
 
-  describe 'Invalid concurrency setting' do
-    it 'raises ArgumentError if concurrency is not positive number' do
-      Shoryuken.options[:concurrency] = -1
-      expect { Shoryuken::Manager.new(nil, nil) }
-        .to raise_error(ArgumentError, 'Concurrency value -1 is invalid, it needs to be a positive number')
-    end
-  end
-
   describe '#start' do
     xit 'pauses when there are no active queues' do
       expect(polling_strategy).to receive(:next_queue).and_return(nil)
@@ -55,6 +47,13 @@ RSpec.describe Shoryuken::Manager do
     end
   end
 
+  describe '#dispatch' do
+    xit 'fires a dispatch event' do
+      expect(subject).to receive(:fire_event).with(:dispatch)
+      subject.send(:dispatch)
+    end
+  end
+
   describe '#dispatch_batch' do
     it 'assings batch as a single message' do
       q = polling_strategy.next_queue

data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb

@@ -34,7 +34,7 @@ RSpec.describe Shoryuken::Middleware::Server::AutoExtendVisibility do
 
   context 'when batch worker' do
     it 'yields' do
-      expect { |b| subject.call(nil, nil, [], nil, &b) }.to yield_control
+      expect { |b| subject.call(TestWorker.new, nil, [], nil, &b) }.to yield_control
     end
   end
 

data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb

@@ -13,7 +13,7 @@ RSpec.describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
 
   context 'when batch worker' do
     it 'yields' do
-      expect { |b| subject.call(nil, nil, [], nil, &b) }.to yield_control
+      expect { |b| subject.call(TestWorker.new, nil, [], nil, &b) }.to yield_control
     end
   end