shoryuken 3.1.1 → 3.2.0

data/lib/shoryuken/options.rb

@@ -21,8 +21,13 @@ module Shoryuken
     @@sqs_client_receive_message_opts = {}
     @@start_callback                  = nil
     @@stop_callback                   = nil
+    @@worker_executor                 = Worker::DefaultExecutor
 
     class << self
+      def active_job?
+        defined?(::ActiveJob)
+      end
+
       def add_group(group, concurrency)
         groups[group] ||= {
           concurrency: concurrency,
@@ -52,8 +57,27 @@ module Shoryuken
         @@worker_registry = worker_registry
       end
 
+      def worker_executor
+        @@worker_executor
+      end
+
+      def worker_executor=(worker_executor)
+        @@worker_executor = worker_executor
+      end
+
       def polling_strategy(group)
-        options[group].to_h.fetch(:polling_strategy, Polling::WeightedRoundRobin)
+        strategy = (group == 'default' ? options : options[:groups].to_h[group]).to_h[:polling_strategy]
+
+        case strategy
+        when 'WeightedRoundRobin', nil # Default case
+          Polling::WeightedRoundRobin
+        when 'StrictPriority'
+          Polling::StrictPriority
+        when Class
+          strategy
+        else
+          raise ArgumentError, "#{strategy} is not a valid polling_strategy"
+        end
       end
 
       def start_callback

data/lib/shoryuken/processor.rb

@@ -4,6 +4,10 @@ module Shoryuken
 
     attr_reader :queue, :sqs_msg
 
+    def self.process(queue, sqs_msg)
+      new(queue, sqs_msg).process
+    end
+
     def initialize(queue, sqs_msg)
       @queue   = queue
       @sqs_msg = sqs_msg
@@ -12,8 +16,10 @@ module Shoryuken
     def process
       return logger.error { "No worker found for #{queue}" } unless worker
 
-      worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
-        worker.perform(sqs_msg, body)
+      Shoryuken::Logging.with_context("#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id}") do
+        worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
+          worker.perform(sqs_msg, body)
+        end
       end
     rescue Exception => ex
       logger.error { "Processor failed: #{ex.message}" }
@@ -37,25 +43,7 @@ module Shoryuken
     end
 
     def parse_body(sqs_msg)
-      body_parser = worker_class.get_shoryuken_options['body_parser']
-
-      case body_parser
-      when :json
-        JSON.parse(sqs_msg.body)
-      when Proc
-        body_parser.call(sqs_msg)
-      when :text, nil
-        sqs_msg.body
-      else
-        if body_parser.respond_to?(:parse)
-          # JSON.parse
-          body_parser.parse(sqs_msg.body)
-        elsif body_parser.respond_to?(:load)
-          # see https://github.com/phstc/shoryuken/pull/91
-          # JSON.load
-          body_parser.load(sqs_msg.body)
-        end
-      end
+      BodyParser.parse(worker_class, sqs_msg)
     end
   end
 end

data/lib/shoryuken/queue.rb

@@ -49,20 +49,27 @@ module Shoryuken
 
     private
 
-    def set_name(name)
+    def set_by_name(name)
       self.name = name
       self.url  = client.get_queue_url(queue_name: name).queue_url
-    rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue => ex
-      raise ex, "The specified queue #{name} does not exist."
     end
 
-    def set_url(url)
+    def set_by_url(url)
       self.name = url.split('/').last
       self.url  = url
     end
 
     def set_name_and_url(name_or_url)
-      name_or_url.start_with?('https://sqs.') ? set_url(name_or_url) : set_name(name_or_url)
+      if name_or_url.start_with?('https://sqs.')
+        set_by_url(name_or_url)
+
+        # anticipate the fifo? checker for validating the queue URL
+        return fifo?
+      end
+
+      set_by_name(name_or_url)
+    rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue => ex
+      raise ex, "The specified queue #{name_or_url} does not exist."
     end
 
     def queue_attributes

data/lib/shoryuken/runner.rb

@@ -118,10 +118,12 @@ module Shoryuken
     end
 
     def handle_signal(sig)
+      logger.debug "Got #{sig} signal"
+
       case sig
       when 'USR1' then execute_soft_shutdown
       when 'TTIN' then print_threads_backtrace
-      else
+      when 'TERM', 'INT'
         logger.info { "Received #{sig}, will shutdown down" }
 
         raise Interrupt

data/lib/shoryuken/util.rb

@@ -4,13 +4,13 @@ module Shoryuken
       Shoryuken.logger
     end
 
-    def fire_event(event, reverse = false)
+    def fire_event(event, reverse = false, event_options = {})
       logger.debug { "Firing '#{event}' lifecycle event" }
       arr = Shoryuken.options[:lifecycle_events][event]
       arr.reverse! if reverse
       arr.each do |block|
         begin
-          block.call
+          block.call(event_options)
         rescue => ex
           logger.warn(event: event)
           logger.warn "#{ex.class.name}: #{ex.message}"
@@ -30,7 +30,7 @@ module Shoryuken
     end
 
     def worker_name(worker_class, sqs_msg, body = nil)
-      if defined?(::ActiveJob) \
+      if Shoryuken.active_job? \
           && !sqs_msg.is_a?(Array) \
           && sqs_msg.message_attributes \
           && sqs_msg.message_attributes['shoryuken_class'] \

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '3.1.1'.freeze
+  VERSION = '3.2.0'.freeze
 end

data/lib/shoryuken/worker/default_executor.rb

@@ -0,0 +1,33 @@
+module Shoryuken
+  module Worker
+    class DefaultExecutor
+      class << self
+        def perform_async(worker_class, body, options = {})
+          options[:message_attributes] ||= {}
+          options[:message_attributes]['shoryuken_class'] = {
+            string_value: worker_class.to_s,
+            data_type: 'String'
+          }
+
+          options[:message_body] = body
+
+          queue = options.delete(:queue) || worker_class.get_shoryuken_options['queue']
+
+          Shoryuken::Client.queues(queue).send_message(options)
+        end
+
+        def perform_in(worker_class, interval, body, options = {})
+          interval = interval.to_f
+          now = Time.now.to_f
+          ts = (interval < 1_000_000_000 ? (now + interval).to_f : interval)
+
+          delay = (ts - now).ceil
+
+          raise 'The maximum allowed delay is 15 minutes' if delay > 15 * 60
+
+          worker_class.perform_async(body, options.merge(delay_seconds: delay))
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/worker/inline_executor.rb

@@ -0,0 +1,28 @@
+module Shoryuken
+  module Worker
+    class InlineExecutor
+      class << self
+        def perform_async(worker_class, body, options = {})
+          body = JSON.dump(body) if body.is_a?(Hash)
+
+          sqs_msg = OpenStruct.new(
+            body: body,
+            attributes: nil,
+            md5_of_body: nil,
+            md5_of_message_attributes: nil,
+            message_attributes: nil,
+            message_id: nil,
+            receipt_handle: nil,
+            delete: nil
+          )
+
+          worker_class.new.perform(sqs_msg, BodyParser.parse(worker_class, sqs_msg))
+        end
+
+        def perform_in(worker_class, _interval, body, options = {})
+          perform_async(worker_class, body, options)
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/worker.rb

@@ -2,33 +2,16 @@ module Shoryuken
   module Worker
     def self.included(base)
       base.extend(ClassMethods)
+      base.shoryuken_class_attribute :shoryuken_options_hash
     end
 
     module ClassMethods
       def perform_async(body, options = {})
-        options[:message_attributes] ||= {}
-        options[:message_attributes]['shoryuken_class'] = {
-          string_value: self.to_s,
-          data_type: 'String'
-        }
-
-        options[:message_body] = body
-
-        queue = options.delete(:queue) || get_shoryuken_options['queue']
-
-        Shoryuken::Client.queues(queue).send_message(options)
+        Shoryuken.worker_executor.perform_async(self, body, options)
       end
 
       def perform_in(interval, body, options = {})
-        interval = interval.to_f
-        now = Time.now.to_f
-        ts = (interval < 1_000_000_000 ? (now + interval).to_f : interval)
-
-        delay = (ts - now).ceil
-
-        raise 'The maximum allowed delay is 15 minutes' if delay > 15 * 60
-
-        perform_async(body, options.merge(delay_seconds: delay))
+        Shoryuken.worker_executor.perform_in(self, interval, body, options)
       end
 
       alias_method :perform_at, :perform_in
@@ -40,7 +23,7 @@ module Shoryuken
       end
 
       def shoryuken_options(opts = {})
-        @shoryuken_options = get_shoryuken_options.merge(stringify_keys(opts || {}))
+        self.shoryuken_options_hash = get_shoryuken_options.merge(stringify_keys(opts || {}))
         normalize_worker_queue!
       end
 
@@ -57,33 +40,87 @@ module Shoryuken
       end
 
       def get_shoryuken_options # :nodoc:
-        @shoryuken_options || Shoryuken.default_worker_options
+        shoryuken_options_hash || Shoryuken.default_worker_options
       end
 
       def stringify_keys(hash) # :nodoc:
-        hash.keys.each do |key|
-          hash[key.to_s] = hash.delete(key)
+        new_hash = {}
+        hash.each { |key, value| new_hash[key.to_s] = value }
+        new_hash
+      end
+
+      def shoryuken_class_attribute(*attrs) # :nodoc:
+        attrs.each do |name|
+          singleton_class.instance_eval do
+            undef_method(name) if method_defined?(name) || private_method_defined?(name)
+          end
+          define_singleton_method(name) { nil }
+
+          ivar = "@#{name}"
+
+          singleton_class.instance_eval do
+            m = "#{name}="
+            undef_method(m) if method_defined?(m) || private_method_defined?(m)
+          end
+
+          define_singleton_method("#{name}=") do |val|
+            singleton_class.class_eval do
+              undef_method(name) if method_defined?(name) || private_method_defined?(name)
+              define_method(name) { val }
+            end
+
+            # singleton? backwards compatibility for ruby < 2.1
+            singleton_klass = respond_to?(:singleton?) ? singleton? : self != ancestors.first
+
+            if singleton_klass
+              class_eval do
+                undef_method(name) if method_defined?(name) || private_method_defined?(name)
+                define_method(name) do
+                  if instance_variable_defined? ivar
+                    instance_variable_get ivar
+                  else
+                    singleton_class.send name
+                  end
+                end
+              end
+            end
+            val
+          end
+
+          # instance reader
+          undef_method(name) if method_defined?(name) || private_method_defined?(name)
+          define_method(name) do
+            if instance_variable_defined?(ivar)
+              instance_variable_get ivar
+            else
+              self.class.public_send name
+            end
+          end
+
+          # instance writer
+          m = "#{name}="
+          undef_method(m) if method_defined?(m) || private_method_defined?(m)
+          attr_writer name
         end
-        hash
       end
 
       private
 
       def normalize_worker_queue!
-        queue = @shoryuken_options['queue']
+        queue = shoryuken_options_hash['queue']
         if queue.respond_to?(:call)
           queue = queue.call
-          @shoryuken_options['queue'] = queue
+          shoryuken_options_hash['queue'] = queue
         end
 
-        case @shoryuken_options['queue']
+        case shoryuken_options_hash['queue']
         when Array
-          @shoryuken_options['queue'].map!(&:to_s)
+          shoryuken_options_hash['queue'].map!(&:to_s)
         when Symbol
-          @shoryuken_options['queue'] = @shoryuken_options['queue'].to_s
+          shoryuken_options_hash['queue'] = shoryuken_options_hash['queue'].to_s
         end
 
-        [@shoryuken_options['queue']].flatten.compact.each(&method(:register_worker))
+        [shoryuken_options_hash['queue']].flatten.compact.each(&method(:register_worker))
       end
 
       def register_worker(queue)

data/lib/shoryuken.rb

@@ -1,8 +1,14 @@
 require 'yaml'
 require 'json'
 require 'aws-sdk-core'
+begin
+  require 'aws-sdk-sqs' unless defined?(Aws::SQS)
+rescue LoadError
+  fail "AWS SDK 3 requires aws-sdk-sqs to be installed separately. Please add gem 'aws-sdk-sqs' to your Gemfile"
+end
 require 'time'
 require 'concurrent'
+require 'forwardable'
 
 require 'shoryuken/version'
 require 'shoryuken/core_ext'
@@ -13,6 +19,8 @@ require 'shoryuken/queue'
 require 'shoryuken/message'
 require 'shoryuken/client'
 require 'shoryuken/worker'
+require 'shoryuken/worker/default_executor'
+require 'shoryuken/worker/inline_executor'
 require 'shoryuken/worker_registry'
 require 'shoryuken/default_worker_registry'
 require 'shoryuken/middleware/chain'
@@ -26,20 +34,25 @@ require 'shoryuken/polling/strict_priority'
 require 'shoryuken/manager'
 require 'shoryuken/launcher'
 require 'shoryuken/processor'
+require 'shoryuken/body_parser'
 require 'shoryuken/fetcher'
 require 'shoryuken/options'
 
+
 module Shoryuken
   extend SingleForwardable
 
   def_delegators(
     :'Shoryuken::Options',
+    :active_job?,
     :add_group,
     :groups,
     :add_queue,
     :ungrouped_queues,
     :worker_registry,
     :worker_registry=,
+    :worker_executor,
+    :worker_executor=,
     :polling_strategy,
     :start_callback,
     :start_callback=,
@@ -67,4 +80,4 @@ module Shoryuken
   )
 end
 
-require 'shoryuken/extensions/active_job_adapter' if defined?(::ActiveJob)
+require 'shoryuken/extensions/active_job_adapter' if Shoryuken.active_job?

data/shoryuken.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'pry-byebug'
   spec.add_development_dependency 'dotenv'
 
-  spec.add_dependency 'aws-sdk-core', '> 2'
+  spec.add_dependency 'aws-sdk-core', '>= 2'
   spec.add_dependency 'concurrent-ruby'
   spec.add_dependency 'thor'
 end

data/spec/shoryuken/body_parser_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+require 'shoryuken/body_parser'
+
+RSpec.describe Shoryuken::BodyParser do
+  let(:sqs_msg) { double }
+
+  describe '#parser' do
+    it 'parses the body into JSON' do
+      TestWorker.get_shoryuken_options['body_parser'] = :json
+
+      body = { 'test' => 'hi' }
+
+      allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
+
+      expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
+    end
+
+    it 'parses the body calling the proc' do
+      TestWorker.get_shoryuken_options['body_parser'] = proc { |sqs_msg| "*#{sqs_msg.body}*" }
+
+      allow(sqs_msg).to receive(:body).and_return('test')
+
+      expect(described_class.parse(TestWorker, sqs_msg)).to eq('*test*')
+    end
+
+    it 'parses the body as text' do
+      TestWorker.get_shoryuken_options['body_parser'] = :text
+
+      body = 'test'
+
+      allow(sqs_msg).to receive(:body).and_return(body)
+
+      expect(described_class.parse(TestWorker, sqs_msg)).to eq('test')
+    end
+
+    it 'parses calling `.load`' do
+      TestWorker.get_shoryuken_options['body_parser'] = Class.new do
+        def self.load(*args)
+          JSON.load(*args)
+        end
+      end
+
+      body = { 'test' => 'hi' }
+
+      allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
+
+      expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
+    end
+
+    it 'parses calling `.parse`' do
+      TestWorker.get_shoryuken_options['body_parser'] = Class.new do
+        def self.parse(*args)
+          JSON.parse(*args)
+        end
+      end
+
+      body = { 'test' => 'hi' }
+
+      allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
+
+      expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
+    end
+
+    context 'when parse errors' do
+      before do
+        TestWorker.get_shoryuken_options['body_parser'] = :json
+
+        allow(sqs_msg).to receive(:body).and_return('invalid JSON')
+      end
+
+      specify do
+        expect { described_class.parse(TestWorker, sqs_msg) }.
+          to raise_error(JSON::ParserError, /unexpected token at 'invalid JSON'/)
+      end
+    end
+
+    context 'when `object_type: nil`' do
+      it 'parses the body as text' do
+        TestWorker.get_shoryuken_options['body_parser'] = nil
+
+        body = 'test'
+
+        allow(sqs_msg).to receive(:body).and_return(body)
+
+        expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
+      end
+    end
+  end
+end

data/spec/shoryuken/environment_loader_spec.rb

@@ -1,13 +1,12 @@
 require 'spec_helper'
+require 'active_job'
 
-# rubocop:disable Metrics/BlockLength
 RSpec.describe Shoryuken::EnvironmentLoader do
   subject { described_class.new({}) }
 
   describe '#parse_queues' do
     before do
-      # TODO proper test other methods
-      allow(subject).to receive(:load_rails).with(anything)
+      allow(subject).to receive(:load_rails)
       allow(subject).to receive(:prefix_active_job_queue_names)
       allow(subject).to receive(:require_workers)
       allow(subject).to receive(:validate_queues)
@@ -22,4 +21,34 @@ RSpec.describe Shoryuken::EnvironmentLoader do
       expect(Shoryuken.groups['default'][:queues]).to eq(%w(queue1 queue2 queue2))
     end
   end
+
+  describe '#prefix_active_job_queue_names' do
+    before do
+      allow(subject).to receive(:load_rails)
+      allow(subject).to receive(:require_workers)
+      allow(subject).to receive(:validate_queues)
+      allow(subject).to receive(:validate_workers)
+      allow(subject).to receive(:patch_deprecated_workers)
+
+      ActiveJob::Base.queue_name_prefix    = 'test'
+      ActiveJob::Base.queue_name_delimiter = '_'
+
+      allow(Shoryuken).to receive(:active_job?).and_return(true)
+    end
+
+    specify do
+      Shoryuken.active_job_queue_name_prefixing = true
+
+      Shoryuken.options[:queues] = ['queue1', ['queue2', 2]]
+
+      Shoryuken.options[:groups] = {
+        'group1' => { queues: %w(group1_queue1 group1_queue2) }
+      }
+
+      subject.load
+
+      expect(Shoryuken.groups['default'][:queues]).to eq(%w(test_queue1 test_queue2 test_queue2))
+      expect(Shoryuken.groups['group1'][:queues]).to eq(%w(test_group1_queue1 test_group1_queue2))
+    end
+  end
 end

data/spec/shoryuken/fetcher_spec.rb

@@ -2,6 +2,7 @@ require 'spec_helper'
 require 'shoryuken/manager'
 require 'shoryuken/fetcher'
 
+# rubocop:disable Metrics/BlockLength
 RSpec.describe Shoryuken::Fetcher do
   let(:queue)        { instance_double('Shoryuken::Queue') }
   let(:queue_name)   { 'default' }
@@ -13,7 +14,7 @@ RSpec.describe Shoryuken::Fetcher do
       Shoryuken::Message,
       queue_url: queue_name,
       body: 'test',
-      message_id: 'fc754df79cc24c4196ca5996a44b771e',
+      message_id: 'fc754df79cc24c4196ca5996a44b771e'
     )
   end
 
@@ -27,23 +28,74 @@ RSpec.describe Shoryuken::Fetcher do
 
       Shoryuken.sqs_client_receive_message_opts[group] = { wait_time_seconds: 10 }
 
-      expect(queue).to receive(:receive_messages).
-        with(wait_time_seconds: 10, max_number_of_messages: limit, message_attribute_names: ['All'], attribute_names: ['All']).
-        and_return([])
+      expect(queue).to receive(:receive_messages).with(
+        wait_time_seconds: 10,
+        max_number_of_messages: limit,
+        message_attribute_names: ['All'],
+        attribute_names: ['All']
+      ).and_return([])
 
       subject.fetch(queue_config, limit)
     end
 
+    it 'logs debug only' do
+      # See https://github.com/phstc/shoryuken/issues/435
+      logger = double 'logger'
+
+      allow(subject).to receive(:logger).and_return(logger)
+
+      expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
+
+      expect(queue).to receive(:receive_messages).and_return([double('SQS Msg')])
+
+      expect(logger).to receive(:debug).exactly(3).times
+      expect(logger).to_not receive(:info)
+
+      subject.fetch(queue_config, limit)
+    end
+
+    context 'when receive options per queue' do
+      let(:limit) { 5 }
+
+      specify do
+        expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
+
+        Shoryuken.sqs_client_receive_message_opts[queue_name] = { max_number_of_messages: 1 }
+
+        expect(queue).to receive(:receive_messages).with(
+          max_number_of_messages: 1,
+          message_attribute_names: ['All'],
+          attribute_names: ['All']
+        ).and_return([])
+
+        subject.fetch(queue_config, limit)
+      end
+    end
+
+    context 'when max_number_of_messages opt is great than limit' do
+      it 'uses limit' do
+        expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
+
+        Shoryuken.sqs_client_receive_message_opts[queue_name] = { max_number_of_messages: 20 }
+
+        expect(queue).to receive(:receive_messages).with(
+          max_number_of_messages: limit,
+          message_attribute_names: ['All'],
+          attribute_names: ['All']
+        ).and_return([])
+
+        subject.fetch(queue_config, limit)
+      end
+    end
+
     context 'when limit is greater than FETCH_LIMIT' do
       let(:limit) { 20 }
 
       specify do
-        Shoryuken.sqs_client_receive_message_opts[group] = {}
-
         allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
-        expect(queue).to receive(:receive_messages).
-          with(max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']).
-          and_return([])
+        expect(queue).to receive(:receive_messages).with(
+          max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']
+        ).and_return([])
 
         subject.fetch(queue_config, limit)
       end