shoryuken 3.0.6 → 4.0.0

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '3.0.6'.freeze
+  VERSION = '4.0.0'.freeze
 end

data/lib/shoryuken/worker/default_executor.rb

@@ -0,0 +1,33 @@
+module Shoryuken
+  module Worker
+    class DefaultExecutor
+      class << self
+        def perform_async(worker_class, body, options = {})
+          options[:message_attributes] ||= {}
+          options[:message_attributes]['shoryuken_class'] = {
+            string_value: worker_class.to_s,
+            data_type: 'String'
+          }
+
+          options[:message_body] = body
+
+          queue = options.delete(:queue) || worker_class.get_shoryuken_options['queue']
+
+          Shoryuken::Client.queues(queue).send_message(options)
+        end
+
+        def perform_in(worker_class, interval, body, options = {})
+          interval = interval.to_f
+          now = Time.now.to_f
+          ts = (interval < 1_000_000_000 ? (now + interval).to_f : interval)
+
+          delay = (ts - now).ceil
+
+          raise 'The maximum allowed delay is 15 minutes' if delay > 15 * 60
+
+          worker_class.perform_async(body, options.merge(delay_seconds: delay))
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/worker/inline_executor.rb

@@ -0,0 +1,37 @@
+module Shoryuken
+  module Worker
+    class InlineExecutor
+      class << self
+        def perform_async(worker_class, body, _options = {})
+          body = JSON.dump(body) if body.is_a?(Hash)
+
+          sqs_msg = OpenStruct.new(
+            body: body,
+            attributes: nil,
+            md5_of_body: nil,
+            md5_of_message_attributes: nil,
+            message_attributes: nil,
+            message_id: nil,
+            receipt_handle: nil,
+            delete: nil
+          )
+
+          call(worker_class, sqs_msg)
+        end
+
+        def perform_in(worker_class, _interval, body, options = {})
+          worker_class.perform_async(body, options)
+        end
+
+        private
+
+        def call(worker_class, sqs_msg)
+          parsed_body = BodyParser.parse(worker_class, sqs_msg)
+          batch = worker_class.shoryuken_options_hash['batch']
+          args = batch ? [[sqs_msg], [parsed_body]] : [sqs_msg, parsed_body]
+          worker_class.new.perform(*args)
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/worker.rb

@@ -2,33 +2,16 @@ module Shoryuken
   module Worker
     def self.included(base)
       base.extend(ClassMethods)
+      base.shoryuken_class_attribute :shoryuken_options_hash
     end
 
     module ClassMethods
       def perform_async(body, options = {})
-        options[:message_attributes] ||= {}
-        options[:message_attributes]['shoryuken_class'] = {
-          string_value: self.to_s,
-          data_type: 'String'
-        }
-
-        options[:message_body] = body
-
-        queue = options.delete(:queue) || get_shoryuken_options['queue']
-
-        Shoryuken::Client.queues(queue).send_message(options)
+        Shoryuken.worker_executor.perform_async(self, body, options)
       end
 
       def perform_in(interval, body, options = {})
-        interval = interval.to_f
-        now = Time.now.to_f
-        ts = (interval < 1_000_000_000 ? (now + interval).to_f : interval)
-
-        delay = (ts - now).ceil
-
-        raise 'The maximum allowed delay is 15 minutes' if delay > 15 * 60
-
-        perform_async(body, options.merge(delay_seconds: delay))
+        Shoryuken.worker_executor.perform_in(self, interval, body, options)
       end
 
       alias_method :perform_at, :perform_in
@@ -40,7 +23,7 @@ module Shoryuken
       end
 
       def shoryuken_options(opts = {})
-        @shoryuken_options = get_shoryuken_options.merge(stringify_keys(opts || {}))
+        self.shoryuken_options_hash = get_shoryuken_options.merge(stringify_keys(opts || {}))
         normalize_worker_queue!
       end
 
@@ -48,34 +31,96 @@ module Shoryuken
         !!get_shoryuken_options['auto_visibility_timeout']
       end
 
+      def exponential_backoff?
+        !!get_shoryuken_options['retry_intervals']
+      end
+
+      def auto_delete?
+        !!(get_shoryuken_options['delete'] || get_shoryuken_options['auto_delete'])
+      end
+
       def get_shoryuken_options # :nodoc:
-        @shoryuken_options || Shoryuken.default_worker_options
+        shoryuken_options_hash || Shoryuken.default_worker_options
       end
 
       def stringify_keys(hash) # :nodoc:
-        hash.keys.each do |key|
-          hash[key.to_s] = hash.delete(key)
+        new_hash = {}
+        hash.each { |key, value| new_hash[key.to_s] = value }
+        new_hash
+      end
+
+      def shoryuken_class_attribute(*attrs) # :nodoc:
+        attrs.each do |name|
+          singleton_class.instance_eval do
+            undef_method(name) if method_defined?(name) || private_method_defined?(name)
+          end
+          define_singleton_method(name) { nil }
+
+          ivar = "@#{name}"
+
+          singleton_class.instance_eval do
+            m = "#{name}="
+            undef_method(m) if method_defined?(m) || private_method_defined?(m)
+          end
+
+          define_singleton_method("#{name}=") do |val|
+            singleton_class.class_eval do
+              undef_method(name) if method_defined?(name) || private_method_defined?(name)
+              define_method(name) { val }
+            end
+
+            # singleton? backwards compatibility for ruby < 2.1
+            singleton_klass = respond_to?(:singleton?) ? singleton? : self != ancestors.first
+
+            if singleton_klass
+              class_eval do
+                undef_method(name) if method_defined?(name) || private_method_defined?(name)
+                define_method(name) do
+                  if instance_variable_defined? ivar
+                    instance_variable_get ivar
+                  else
+                    singleton_class.send name
+                  end
+                end
+              end
+            end
+            val
+          end
+
+          # instance reader
+          undef_method(name) if method_defined?(name) || private_method_defined?(name)
+          define_method(name) do
+            if instance_variable_defined?(ivar)
+              instance_variable_get ivar
+            else
+              self.class.public_send name
+            end
+          end
+
+          # instance writer
+          m = "#{name}="
+          undef_method(m) if method_defined?(m) || private_method_defined?(m)
+          attr_writer name
         end
-        hash
       end
 
       private
 
       def normalize_worker_queue!
-        queue = @shoryuken_options['queue']
+        queue = shoryuken_options_hash['queue']
         if queue.respond_to?(:call)
           queue = queue.call
-          @shoryuken_options['queue'] = queue
+          shoryuken_options_hash['queue'] = queue
         end
 
-        case @shoryuken_options['queue']
+        case shoryuken_options_hash['queue']
         when Array
-          @shoryuken_options['queue'].map!(&:to_s)
+          shoryuken_options_hash['queue'].map!(&:to_s)
         when Symbol
-          @shoryuken_options['queue'] = @shoryuken_options['queue'].to_s
+          shoryuken_options_hash['queue'] = shoryuken_options_hash['queue'].to_s
         end
 
-        [@shoryuken_options['queue']].flatten.compact.each(&method(:register_worker))
+        [shoryuken_options_hash['queue']].flatten.compact.each(&method(:register_worker))
       end
 
       def register_worker(queue)

data/lib/shoryuken/worker_registry.rb

@@ -1,6 +1,6 @@
 module Shoryuken
   class WorkerRegistry
-    def batch_receive_messages?(queue)
+    def batch_receive_messages?(_queue)
       # true if the workers for queue support batch processing of messages
       fail NotImplementedError
     end
@@ -10,7 +10,7 @@ module Shoryuken
       fail NotImplementedError
     end
 
-    def fetch_worker(queue, message)
+    def fetch_worker(_queue, _message)
       # must return an instance of the worker that handles
       # message received on queue
       fail NotImplementedError
@@ -21,12 +21,12 @@ module Shoryuken
       fail NotImplementedError
     end
 
-    def register_worker(queue, clazz)
+    def register_worker(_queue, _clazz)
       # must register the worker as a consumer of messages from queue
       fail NotImplementedError
     end
 
-    def workers(queue)
+    def workers(_queue)
       # must return the list of workers registered for queue, or []
       fail NotImplementedError
     end

data/lib/shoryuken.rb

@@ -1,8 +1,14 @@
 require 'yaml'
 require 'json'
 require 'aws-sdk-core'
+begin
+  require 'aws-sdk-sqs' unless defined?(Aws::SQS)
+rescue LoadError
+  fail "AWS SDK 3 requires aws-sdk-sqs to be installed separately. Please add gem 'aws-sdk-sqs' to your Gemfile"
+end
 require 'time'
 require 'concurrent'
+require 'forwardable'
 
 require 'shoryuken/version'
 require 'shoryuken/core_ext'
@@ -13,6 +19,8 @@ require 'shoryuken/queue'
 require 'shoryuken/message'
 require 'shoryuken/client'
 require 'shoryuken/worker'
+require 'shoryuken/worker/default_executor'
+require 'shoryuken/worker/inline_executor'
 require 'shoryuken/worker_registry'
 require 'shoryuken/default_worker_registry'
 require 'shoryuken/middleware/chain'
@@ -20,184 +28,57 @@ require 'shoryuken/middleware/server/auto_delete'
 Shoryuken::Middleware::Server.autoload :AutoExtendVisibility, 'shoryuken/middleware/server/auto_extend_visibility'
 require 'shoryuken/middleware/server/exponential_backoff_retry'
 require 'shoryuken/middleware/server/timing'
-require 'shoryuken/polling'
+require 'shoryuken/polling/base'
+require 'shoryuken/polling/weighted_round_robin'
+require 'shoryuken/polling/strict_priority'
 require 'shoryuken/manager'
 require 'shoryuken/launcher'
 require 'shoryuken/processor'
+require 'shoryuken/body_parser'
 require 'shoryuken/fetcher'
+require 'shoryuken/options'
 
 module Shoryuken
-  DEFAULTS = {
-    concurrency: 25,
-    queues: [],
-    aws: {},
-    delay: 0,
-    timeout: 8,
-    lifecycle_events: {
-      startup: [],
-      quiet: [],
-      shutdown: []
-    },
-    polling_strategy: Polling::WeightedRoundRobin
-  }.freeze
-
-  @@queues                          = []
-  @@worker_registry                 = DefaultWorkerRegistry.new
-  @@active_job_queue_name_prefixing = false
-  @@sqs_client                      = nil
-  @@sqs_client_receive_message_opts = {}
-  @@start_callback                  = nil
-  @@stop_callback                   = nil
-
-  class << self
-    def queues
-      @@queues
-    end
-
-    def add_queue(queue, priority = 1)
-      priority.times { queues << queue }
-    end
-
-    def worker_registry
-      @@worker_registry
-    end
-
-    def worker_registry=(worker_registry)
-      @@worker_registry = worker_registry
-    end
-
-    def start_callback
-      @@start_callback
-    end
-
-    def start_callback=(start_callback)
-      @@start_callback = start_callback
-    end
-
-    def stop_callback
-      @@stop_callback
-    end
-
-    def stop_callback=(stop_callback)
-      @@stop_callback = stop_callback
-    end
-
-    def active_job_queue_name_prefixing
-      @@active_job_queue_name_prefixing
-    end
-
-    def active_job_queue_name_prefixing=(active_job_queue_name_prefixing)
-      @@active_job_queue_name_prefixing = active_job_queue_name_prefixing
-    end
-
-    def sqs_client
-      @@sqs_client ||= Aws::SQS::Client.new
-    end
-
-    def sqs_client=(sqs_client)
-      @@sqs_client = sqs_client
-    end
-
-    def sqs_client_receive_message_opts
-      @@sqs_client_receive_message_opts
-    end
-
-    def sqs_client_receive_message_opts=(sqs_client_receive_message_opts)
-      @@sqs_client_receive_message_opts = sqs_client_receive_message_opts
-    end
-
-    def options
-      @@options ||= DEFAULTS.dup
-    end
-
-    def logger
-      Shoryuken::Logging.logger
-    end
-
-    def register_worker(*args)
-      @@worker_registry.register_worker(*args)
-    end
-
-    def configure_server
-      yield self if server?
-    end
-
-    def server_middleware
-      @@server_chain ||= default_server_middleware
-      yield @@server_chain if block_given?
-      @@server_chain
-    end
-
-    def configure_client
-      yield self unless server?
-    end
-
-    def client_middleware
-      @@client_chain ||= default_client_middleware
-      yield @@client_chain if block_given?
-      @@client_chain
-    end
-
-    def default_worker_options
-      @@default_worker_options ||= {
-        'queue'                   => 'default',
-        'delete'                  => false,
-        'auto_delete'             => false,
-        'auto_visibility_timeout' => false,
-        'retry_intervals'         => nil,
-        'batch'                   => false
-      }
-    end
-
-    def default_worker_options=(default_worker_options)
-      @@default_worker_options = default_worker_options
-    end
-
-    def on_start(&block)
-      @@start_callback = block
-    end
-
-    def on_stop(&block)
-      @@stop_callback = block
-    end
-
-    # Register a block to run at a point in the Shoryuken lifecycle.
-    # :startup, :quiet or :shutdown are valid events.
-    #
-    #   Shoryuken.configure_server do |config|
-    #     config.on(:shutdown) do
-    #       puts "Goodbye cruel world!"
-    #     end
-    #   end
-    def on(event, &block)
-      fail ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
-      fail ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
-      options[:lifecycle_events][event] << block
-    end
-
-    private
-
-    def default_server_middleware
-      Middleware::Chain.new do |m|
-        m.add Middleware::Server::Timing
-        m.add Middleware::Server::ExponentialBackoffRetry
-        m.add Middleware::Server::AutoDelete
-        m.add Middleware::Server::AutoExtendVisibility
-        if defined?(::ActiveRecord::Base)
-          require 'shoryuken/middleware/server/active_record'
-          m.add Middleware::Server::ActiveRecord
-        end
-      end
-    end
-
-    def default_client_middleware
-      Middleware::Chain.new
-    end
-
-    def server?
-      defined?(Shoryuken::CLI)
-    end
-  end
+  extend SingleForwardable
+
+  def_delegators(
+    :'Shoryuken::Options',
+    :active_job?,
+    :add_group,
+    :groups,
+    :add_queue,
+    :ungrouped_queues,
+    :worker_registry,
+    :worker_registry=,
+    :worker_executor,
+    :worker_executor=,
+    :launcher_executor,
+    :launcher_executor=,
+    :polling_strategy,
+    :start_callback,
+    :start_callback=,
+    :stop_callback,
+    :stop_callback=,
+    :active_job_queue_name_prefixing,
+    :active_job_queue_name_prefixing=,
+    :sqs_client,
+    :sqs_client=,
+    :sqs_client_receive_message_opts,
+    :sqs_client_receive_message_opts=,
+    :options,
+    :logger,
+    :register_worker,
+    :configure_server,
+    :server?,
+    :server_middleware,
+    :configure_client,
+    :client_middleware,
+    :default_worker_options,
+    :default_worker_options=,
+    :on_start,
+    :on_stop,
+    :on
+  )
 end
 
-require 'shoryuken/extensions/active_job_adapter' if defined?(::ActiveJob)
+require 'shoryuken/extensions/active_job_adapter' if Shoryuken.active_job?

data/shoryuken.gemspec

@@ -1,5 +1,5 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'shoryuken/version'
 
@@ -13,17 +13,17 @@ Gem::Specification.new do |spec|
   spec.license       = 'LGPL-3.0'
 
   spec.files         = `git ls-files -z`.split("\x0")
-  spec.executables   = %w(shoryuken)
+  spec.executables   = %w[shoryuken]
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'dotenv'
+  spec.add_development_dependency 'pry-byebug'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
-  spec.add_development_dependency 'pry-byebug'
-  spec.add_development_dependency 'dotenv'
 
-  spec.add_dependency 'aws-sdk-core', '~> 2'
+  spec.add_dependency 'aws-sdk-core', '>= 2'
   spec.add_dependency 'concurrent-ruby'
   spec.add_dependency 'thor'
 end

data/spec/integration/launcher_spec.rb

@@ -4,27 +4,33 @@ require 'shoryuken/launcher'
 require 'securerandom'
 
 RSpec.describe Shoryuken::Launcher do
-  describe 'Consuming messages', slow: :true do
+  describe 'Consuming messages', slow: true do
     before do
+      Aws.config[:stub_responses] = false
+      Aws.config[:region] = 'us-east-1'
+
       StandardWorker.received_messages = 0
 
-      queue = "test_shoryuken#{StandardWorker}_#{SecureRandom.uuid}"
+      queue = "shoryuken-travis-#{StandardWorker}-#{SecureRandom.uuid}"
 
-      Shoryuken::Client.sqs.create_queue queue_name: queue
+      Shoryuken::Client.sqs.create_queue(queue_name: queue)
 
-      Shoryuken.queues << queue
+      Shoryuken.add_group('default', 1)
+      Shoryuken.add_queue(queue, 1, 'default')
 
       StandardWorker.get_shoryuken_options['queue'] = queue
 
-      Shoryuken.register_worker queue, StandardWorker
+      Shoryuken.register_worker(queue, StandardWorker)
     end
 
     after do
+      Aws.config[:stub_responses] = true
+
       queue_url = Shoryuken::Client.sqs.get_queue_url(
         queue_name: StandardWorker.get_shoryuken_options['queue']
       ).queue_url
 
-      Shoryuken::Client.sqs.delete_queue queue_url: queue_url
+      Shoryuken::Client.sqs.delete_queue(queue_url: queue_url)
     end
 
     it 'consumes as a command worker' do
@@ -61,7 +67,7 @@ RSpec.describe Shoryuken::Launcher do
     end
 
     def poll_queues_until
-      subject.run
+      subject.start
 
       Timeout::timeout(10) do
         begin
@@ -79,7 +85,7 @@ RSpec.describe Shoryuken::Launcher do
 
       shoryuken_options auto_delete: true
 
-      def perform(sqs_msg, body)
+      def perform(sqs_msg, _body)
         @@received_messages += Array(sqs_msg).size
       end
 

data/spec/shoryuken/body_parser_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+require 'shoryuken/body_parser'
+
+RSpec.describe Shoryuken::BodyParser do
+  let(:sqs_msg) { double }
+
+  describe '#parser' do
+    it 'parses the body into JSON' do
+      TestWorker.get_shoryuken_options['body_parser'] = :json
+
+      body = { 'test' => 'hi' }
+
+      allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
+
+      expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
+    end
+
+    it 'parses the body calling the proc' do
+      TestWorker.get_shoryuken_options['body_parser'] = proc { |sqs_msg| "*#{sqs_msg.body}*" }
+
+      allow(sqs_msg).to receive(:body).and_return('test')
+
+      expect(described_class.parse(TestWorker, sqs_msg)).to eq('*test*')
+    end
+
+    it 'parses the body as text' do
+      TestWorker.get_shoryuken_options['body_parser'] = :text
+
+      body = 'test'
+
+      allow(sqs_msg).to receive(:body).and_return(body)
+
+      expect(described_class.parse(TestWorker, sqs_msg)).to eq('test')
+    end
+
+    it 'parses calling `.load`' do
+      TestWorker.get_shoryuken_options['body_parser'] = Class.new do
+        def self.load(*args)
+          JSON.load(*args)
+        end
+      end
+
+      body = { 'test' => 'hi' }
+
+      allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
+
+      expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
+    end
+
+    it 'parses calling `.parse`' do
+      TestWorker.get_shoryuken_options['body_parser'] = Class.new do
+        def self.parse(*args)
+          JSON.parse(*args)
+        end
+      end
+
+      body = { 'test' => 'hi' }
+
+      allow(sqs_msg).to receive(:body).and_return(JSON.dump(body))
+
+      expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
+    end
+
+    context 'when parse errors' do
+      before do
+        TestWorker.get_shoryuken_options['body_parser'] = :json
+
+        allow(sqs_msg).to receive(:body).and_return('invalid JSON')
+      end
+
+      specify do
+        expect { described_class.parse(TestWorker, sqs_msg) }
+          .to raise_error(JSON::ParserError, /unexpected token at 'invalid JSON'/)
+      end
+    end
+
+    context 'when `object_type: nil`' do
+      it 'parses the body as text' do
+        TestWorker.get_shoryuken_options['body_parser'] = nil
+
+        body = 'test'
+
+        allow(sqs_msg).to receive(:body).and_return(body)
+
+        expect(described_class.parse(TestWorker, sqs_msg)).to eq(body)
+      end
+    end
+  end
+end