shoryuken 3.1.8 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b0666812a472f98087bf90ff23f4420b0ba78bb9
-  data.tar.gz: fe0c97e16e371afed0895bc8560d77503c2c8962
+  metadata.gz: f74e5fbb599fe5bbe41bed5fd542c0c1fdcc2d35
+  data.tar.gz: 6099d42682b5e2a1797d7cdbdc7bf27b344b692c
 SHA512:
-  metadata.gz: ee9e3b82feb7124d3ef165c632c0b2078d22be7939bb2eaef7da036812859a13529f08c09b2b6c1b0eccf24f1c06ed6be9310415e894d59d1363d7574ce5764f
-  data.tar.gz: 65b44b013c44cf6521ab9cb6a4d21095d6475ba2cf8bb196436807ec0269fe56ef3e6ddd7a3d0efd0cdf04c2408f06fdea89688aea1da12ef1e609d413f2eed2
+  metadata.gz: f076e8f427f39ca92be4f27be819350d90270ce9e695d1173a6944b82d16680d11afd78a0075dd549e0a2c6031e6d5708c306cb245c4d7dfcc69d8a5e7151bc7
+  data.tar.gz: 2fd34c84e6260df3c35427ffb27d3d067fc1ea11469609d61e8eedd465e4516d75e052079b5195ede7c7917ad3c97bb39c5440bc5436ae3ab2581f3fb72bad31

data/.rubocop.yml

@@ -1,5 +1,5 @@
 AllCops:
-  TargetRubyVersion: 2.0
+  TargetRubyVersion: 2.1
 
 Style/SignalException:
   Enabled: false

data/.travis.yml

@@ -1,4 +1,5 @@
 language: ruby
+
 rvm:
   - 2.0.0
   - 2.1.10
@@ -11,9 +12,18 @@ notifications:
     on_success: change
     on_failure: always
 
-script: SPEC_ALL=true bundle exec rspec spec
+gemfile:
+  - Gemfile
+  - Gemfile.aws-sdk-core-v2
+
+env:
+  - SPEC_ALL=true
+
+script: bundle exec rspec spec
+
 before_install:
- - gem update bundler
+  - gem update bundler
+
 after_success:
   - bundle exec codeclimate-test-reporter
 

data/CHANGELOG.md

@@ -1,3 +1,37 @@
+## [v3.2.0] - 2018-01-03
+
+- Preserve parent worker class options
+ - [#451](https://github.com/phstc/shoryuken/pull/451)
+
+- Add -t (shutdown timeout) option to CL
+ - [#449](https://github.com/phstc/shoryuken/pull/449)
+
+- Support inline (Active Job like) for standard workers
+ - [#448](https://github.com/phstc/shoryuken/pull/448)
+
+## [v3.1.12] - 2017-09-25
+
+- Reduce fetch log verbosity
+ - [#436](https://github.com/phstc/shoryuken/pull/436)
+
+## [v3.1.11] - 2017-09-02
+
+- Auto retry (up to 3 times) fetch errors
+ - [#429](https://github.com/phstc/shoryuken/pull/429)
+
+## [v3.1.10] - 2017-09-02
+
+- Make Shoryuken compatible with AWS SDK 3 and 2
+ - [#433](https://github.com/phstc/shoryuken/pull/433)
+
+## [v3.1.9] - 2017-08-24
+
+- Add support for adding a middleware to the front of chain
+ - [#427](https://github.com/phstc/shoryuken/pull/427)
+
+- Add support for dispatch fire event
+ - [#426](https://github.com/phstc/shoryuken/pull/426)
+
 ## [v3.1.8] - 2017-08-17
 
 - Make Polling strategy backward compatibility

data/Gemfile

@@ -4,9 +4,11 @@ source 'https://rubygems.org'
 gemspec
 
 group :test do
-  gem 'activejob'
+  gem 'activejob', '~> 4'
+  gem 'aws-sdk-core', '~> 3'
+  gem 'aws-sdk-sqs'
   gem 'codeclimate-test-reporter', require: nil
-  gem 'simplecov'
-  gem 'multi_xml'
   gem 'httparty'
+  gem 'multi_xml'
+  gem 'simplecov'
 end

data/Gemfile.aws-sdk-core-v2

@@ -0,0 +1,13 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in shoryuken.gemspec
+gemspec
+
+group :test do
+  gem 'activejob', '~> 4'
+  gem 'aws-sdk-core', '~> 2'
+  gem 'codeclimate-test-reporter', require: nil
+  gem 'httparty'
+  gem 'multi_xml'
+  gem 'simplecov'
+end

data/bin/shoryuken

@@ -21,6 +21,7 @@ module Shoryuken
       method_option :daemon,      aliases: '-d', type: :boolean, desc: 'Daemonize process'
       method_option :queues,      aliases: '-q', type: :array,   desc: 'Queues to process with optional weights'
       method_option :require,     aliases: '-r', type: :string,  desc: 'Dir or path of the workers'
+      method_option :timeout,     aliases: '-t', type: :numeric, desc: 'Hard shutdown timeout'
       method_option :config,      aliases: '-C', type: :string,  desc: 'Path to config file'
       method_option :config_file,                type: :string,  desc: 'Path to config file (backwards compatibility)'
       method_option :rails,       aliases: '-R', type: :boolean, desc: 'Load Rails'

data/lib/shoryuken/body_parser.rb

@@ -0,0 +1,27 @@
+module Shoryuken
+  class BodyParser
+    class << self
+      def parse(worker_class, sqs_msg)
+        body_parser = worker_class.get_shoryuken_options['body_parser']
+
+        case body_parser
+        when :json
+          JSON.parse(sqs_msg.body)
+        when Proc
+          body_parser.call(sqs_msg)
+        when :text, nil
+          sqs_msg.body
+        else
+          if body_parser.respond_to?(:parse)
+            # JSON.parse
+            body_parser.parse(sqs_msg.body)
+          elsif body_parser.respond_to?(:load)
+            # see https://github.com/phstc/shoryuken/pull/91
+            # JSON.load
+            body_parser.load(sqs_msg.body)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/fetcher.rb

@@ -9,20 +9,41 @@ module Shoryuken
     end
 
     def fetch(queue, limit)
-      started_at = Time.now
+      fetch_with_auto_retry(3) do
+        started_at = Time.now
 
-      logger.debug { "Looking for new messages in #{queue}" }
+        logger.debug { "Looking for new messages in #{queue}" }
 
-      sqs_msgs = Array(receive_messages(queue, [FETCH_LIMIT, limit].min))
+        sqs_msgs = Array(receive_messages(queue, [FETCH_LIMIT, limit].min))
 
-      logger.info { "Found #{sqs_msgs.size} messages for #{queue.name}" } unless sqs_msgs.empty?
-      logger.debug { "Fetcher for #{queue} completed in #{elapsed(started_at)} ms" }
+        logger.debug { "Found #{sqs_msgs.size} messages for #{queue.name}" } unless sqs_msgs.empty?
+        logger.debug { "Fetcher for #{queue} completed in #{elapsed(started_at)} ms" }
 
-      sqs_msgs
+        sqs_msgs
+      end
     end
 
     private
 
+    def fetch_with_auto_retry(max_attempts, &block)
+      attempts = 0
+
+      begin
+        yield
+      rescue => ex
+        # Tries to auto retry connectivity errors
+        raise if attempts >= max_attempts
+
+        attempts += 1
+
+        logger.debug { "Retrying fetch attempt #{attempts} for #{ex.message}" }
+
+        sleep((1..5).to_a.sample)
+
+        retry
+      end
+    end
+
     def receive_messages(queue, limit)
       options = receive_options(queue)
 

data/lib/shoryuken/manager.rb

@@ -12,6 +12,7 @@ module Shoryuken
       @max_processors   = concurrency
       @busy_processors  = Concurrent::AtomicFixnum.new(0)
       @executor         = executor
+      @running          = Concurrent::AtomicBoolean.new(true)
     end
 
     def start
@@ -21,7 +22,7 @@ module Shoryuken
     private
 
     def running?
-      @executor.running?
+      @running.true? && @executor.running?
     end
 
     def dispatch_loop
@@ -37,7 +38,7 @@ module Shoryuken
         return sleep(MIN_DISPATCH_INTERVAL)
       end
 
-      fire_event(:dispatch)
+      fire_event(:dispatch, false, queue_name: queue.name)
 
       logger.debug { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
 
@@ -104,6 +105,8 @@ module Shoryuken
       logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
 
       Process.kill('USR1', Process.pid)
+
+      @running.make_false
     end
   end
 end

data/lib/shoryuken/middleware/chain.rb

@@ -61,6 +61,10 @@ module Shoryuken
         entries << Entry.new(klass, *args) unless exists?(klass)
       end
 
+      def prepend(klass, *args)
+        entries.insert(0, Entry.new(klass, *args)) unless exists?(klass)
+      end
+
       def insert_before(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)

data/lib/shoryuken/middleware/server/auto_extend_visibility.rb

@@ -31,16 +31,13 @@ module Shoryuken
             Concurrent::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
               begin
                 logger.debug do
-                  "Extending message #{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id} " \
-                  "visibility timeout by #{queue_visibility_timeout}s."
+                  "Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
                 end
 
                 sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
               rescue => ex
                 logger.error do
-                  'Could not auto extend the message ' \
-                  "#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id} " \
-                  "visibility timeout. Error: #{ex.message}"
+                  "Could not auto extend the message #{queue}/#{sqs_msg.message_id} visibility timeout. Error: #{ex.message}"
                 end
               end
             end

data/lib/shoryuken/middleware/server/timing.rb

@@ -5,25 +5,23 @@ module Shoryuken
         include Util
 
         def call(worker, queue, sqs_msg, body)
-          Shoryuken::Logging.with_context("#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id}") do
-            begin
-              started_at = Time.now
+          begin
+            started_at = Time.now
 
-              logger.info { "started at #{started_at}" }
+            logger.info { "started at #{started_at}" }
 
-              yield
+            yield
 
-              total_time = elapsed(started_at)
+            total_time = elapsed(started_at)
 
-              if (total_time / 1000.0) > (timeout = Shoryuken::Client.queues(queue).visibility_timeout)
-                logger.warn { "exceeded the queue visibility timeout by #{total_time - (timeout * 1000)} ms" }
-              end
-
-              logger.info { "completed in: #{total_time} ms" }
-            rescue
-              logger.info { "failed in: #{elapsed(started_at)} ms" }
-              raise
+            if (total_time / 1000.0) > (timeout = Shoryuken::Client.queues(queue).visibility_timeout)
+              logger.warn { "exceeded the queue visibility timeout by #{total_time - (timeout * 1000)} ms" }
             end
+
+            logger.info { "completed in: #{total_time} ms" }
+          rescue
+            logger.info { "failed in: #{elapsed(started_at)} ms" }
+            raise
           end
         end
       end

data/lib/shoryuken/options.rb

@@ -21,6 +21,7 @@ module Shoryuken
     @@sqs_client_receive_message_opts = {}
     @@start_callback                  = nil
     @@stop_callback                   = nil
+    @@worker_executor                 = Worker::DefaultExecutor
 
     class << self
       def active_job?
@@ -56,6 +57,14 @@ module Shoryuken
         @@worker_registry = worker_registry
       end
 
+      def worker_executor
+        @@worker_executor
+      end
+
+      def worker_executor=(worker_executor)
+        @@worker_executor = worker_executor
+      end
+
       def polling_strategy(group)
         strategy = (group == 'default' ? options : options[:groups].to_h[group]).to_h[:polling_strategy]
 

data/lib/shoryuken/processor.rb

@@ -16,8 +16,10 @@ module Shoryuken
     def process
       return logger.error { "No worker found for #{queue}" } unless worker
 
-      worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
-        worker.perform(sqs_msg, body)
+      Shoryuken::Logging.with_context("#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id}") do
+        worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
+          worker.perform(sqs_msg, body)
+        end
       end
     rescue Exception => ex
       logger.error { "Processor failed: #{ex.message}" }
@@ -41,25 +43,7 @@ module Shoryuken
     end
 
     def parse_body(sqs_msg)
-      body_parser = worker_class.get_shoryuken_options['body_parser']
-
-      case body_parser
-      when :json
-        JSON.parse(sqs_msg.body)
-      when Proc
-        body_parser.call(sqs_msg)
-      when :text, nil
-        sqs_msg.body
-      else
-        if body_parser.respond_to?(:parse)
-          # JSON.parse
-          body_parser.parse(sqs_msg.body)
-        elsif body_parser.respond_to?(:load)
-          # see https://github.com/phstc/shoryuken/pull/91
-          # JSON.load
-          body_parser.load(sqs_msg.body)
-        end
-      end
+      BodyParser.parse(worker_class, sqs_msg)
     end
   end
 end

data/lib/shoryuken/util.rb

@@ -4,13 +4,13 @@ module Shoryuken
       Shoryuken.logger
     end
 
-    def fire_event(event, reverse = false)
+    def fire_event(event, reverse = false, event_options = {})
       logger.debug { "Firing '#{event}' lifecycle event" }
       arr = Shoryuken.options[:lifecycle_events][event]
       arr.reverse! if reverse
       arr.each do |block|
         begin
-          block.call
+          block.call(event_options)
         rescue => ex
           logger.warn(event: event)
           logger.warn "#{ex.class.name}: #{ex.message}"

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '3.1.8'.freeze
+  VERSION = '3.2.0'.freeze
 end

data/lib/shoryuken/worker/default_executor.rb

@@ -0,0 +1,33 @@
+module Shoryuken
+  module Worker
+    class DefaultExecutor
+      class << self
+        def perform_async(worker_class, body, options = {})
+          options[:message_attributes] ||= {}
+          options[:message_attributes]['shoryuken_class'] = {
+            string_value: worker_class.to_s,
+            data_type: 'String'
+          }
+
+          options[:message_body] = body
+
+          queue = options.delete(:queue) || worker_class.get_shoryuken_options['queue']
+
+          Shoryuken::Client.queues(queue).send_message(options)
+        end
+
+        def perform_in(worker_class, interval, body, options = {})
+          interval = interval.to_f
+          now = Time.now.to_f
+          ts = (interval < 1_000_000_000 ? (now + interval).to_f : interval)
+
+          delay = (ts - now).ceil
+
+          raise 'The maximum allowed delay is 15 minutes' if delay > 15 * 60
+
+          worker_class.perform_async(body, options.merge(delay_seconds: delay))
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/worker/inline_executor.rb

@@ -0,0 +1,28 @@
+module Shoryuken
+  module Worker
+    class InlineExecutor
+      class << self
+        def perform_async(worker_class, body, options = {})
+          body = JSON.dump(body) if body.is_a?(Hash)
+
+          sqs_msg = OpenStruct.new(
+            body: body,
+            attributes: nil,
+            md5_of_body: nil,
+            md5_of_message_attributes: nil,
+            message_attributes: nil,
+            message_id: nil,
+            receipt_handle: nil,
+            delete: nil
+          )
+
+          worker_class.new.perform(sqs_msg, BodyParser.parse(worker_class, sqs_msg))
+        end
+
+        def perform_in(worker_class, _interval, body, options = {})
+          perform_async(worker_class, body, options)
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/worker.rb

@@ -2,33 +2,16 @@ module Shoryuken
   module Worker
     def self.included(base)
       base.extend(ClassMethods)
+      base.shoryuken_class_attribute :shoryuken_options_hash
     end
 
     module ClassMethods
       def perform_async(body, options = {})
-        options[:message_attributes] ||= {}
-        options[:message_attributes]['shoryuken_class'] = {
-          string_value: self.to_s,
-          data_type: 'String'
-        }
-
-        options[:message_body] = body
-
-        queue = options.delete(:queue) || get_shoryuken_options['queue']
-
-        Shoryuken::Client.queues(queue).send_message(options)
+        Shoryuken.worker_executor.perform_async(self, body, options)
       end
 
       def perform_in(interval, body, options = {})
-        interval = interval.to_f
-        now = Time.now.to_f
-        ts = (interval < 1_000_000_000 ? (now + interval).to_f : interval)
-
-        delay = (ts - now).ceil
-
-        raise 'The maximum allowed delay is 15 minutes' if delay > 15 * 60
-
-        perform_async(body, options.merge(delay_seconds: delay))
+        Shoryuken.worker_executor.perform_in(self, interval, body, options)
       end
 
       alias_method :perform_at, :perform_in
@@ -40,7 +23,7 @@ module Shoryuken
       end
 
       def shoryuken_options(opts = {})
-        @shoryuken_options = get_shoryuken_options.merge(stringify_keys(opts || {}))
+        self.shoryuken_options_hash = get_shoryuken_options.merge(stringify_keys(opts || {}))
         normalize_worker_queue!
       end
 
@@ -57,33 +40,87 @@ module Shoryuken
       end
 
       def get_shoryuken_options # :nodoc:
-        @shoryuken_options || Shoryuken.default_worker_options
+        shoryuken_options_hash || Shoryuken.default_worker_options
       end
 
       def stringify_keys(hash) # :nodoc:
-        hash.keys.each do |key|
-          hash[key.to_s] = hash.delete(key)
+        new_hash = {}
+        hash.each { |key, value| new_hash[key.to_s] = value }
+        new_hash
+      end
+
+      def shoryuken_class_attribute(*attrs) # :nodoc:
+        attrs.each do |name|
+          singleton_class.instance_eval do
+            undef_method(name) if method_defined?(name) || private_method_defined?(name)
+          end
+          define_singleton_method(name) { nil }
+
+          ivar = "@#{name}"
+
+          singleton_class.instance_eval do
+            m = "#{name}="
+            undef_method(m) if method_defined?(m) || private_method_defined?(m)
+          end
+
+          define_singleton_method("#{name}=") do |val|
+            singleton_class.class_eval do
+              undef_method(name) if method_defined?(name) || private_method_defined?(name)
+              define_method(name) { val }
+            end
+
+            # singleton? backwards compatibility for ruby < 2.1
+            singleton_klass = respond_to?(:singleton?) ? singleton? : self != ancestors.first
+
+            if singleton_klass
+              class_eval do
+                undef_method(name) if method_defined?(name) || private_method_defined?(name)
+                define_method(name) do
+                  if instance_variable_defined? ivar
+                    instance_variable_get ivar
+                  else
+                    singleton_class.send name
+                  end
+                end
+              end
+            end
+            val
+          end
+
+          # instance reader
+          undef_method(name) if method_defined?(name) || private_method_defined?(name)
+          define_method(name) do
+            if instance_variable_defined?(ivar)
+              instance_variable_get ivar
+            else
+              self.class.public_send name
+            end
+          end
+
+          # instance writer
+          m = "#{name}="
+          undef_method(m) if method_defined?(m) || private_method_defined?(m)
+          attr_writer name
         end
-        hash
       end
 
       private
 
       def normalize_worker_queue!
-        queue = @shoryuken_options['queue']
+        queue = shoryuken_options_hash['queue']
         if queue.respond_to?(:call)
           queue = queue.call
-          @shoryuken_options['queue'] = queue
+          shoryuken_options_hash['queue'] = queue
         end
 
-        case @shoryuken_options['queue']
+        case shoryuken_options_hash['queue']
         when Array
-          @shoryuken_options['queue'].map!(&:to_s)
+          shoryuken_options_hash['queue'].map!(&:to_s)
         when Symbol
-          @shoryuken_options['queue'] = @shoryuken_options['queue'].to_s
+          shoryuken_options_hash['queue'] = shoryuken_options_hash['queue'].to_s
         end
 
-        [@shoryuken_options['queue']].flatten.compact.each(&method(:register_worker))
+        [shoryuken_options_hash['queue']].flatten.compact.each(&method(:register_worker))
       end
 
       def register_worker(queue)

data/lib/shoryuken.rb

@@ -1,6 +1,11 @@
 require 'yaml'
 require 'json'
 require 'aws-sdk-core'
+begin
+  require 'aws-sdk-sqs' unless defined?(Aws::SQS)
+rescue LoadError
+  fail "AWS SDK 3 requires aws-sdk-sqs to be installed separately. Please add gem 'aws-sdk-sqs' to your Gemfile"
+end
 require 'time'
 require 'concurrent'
 require 'forwardable'
@@ -14,6 +19,8 @@ require 'shoryuken/queue'
 require 'shoryuken/message'
 require 'shoryuken/client'
 require 'shoryuken/worker'
+require 'shoryuken/worker/default_executor'
+require 'shoryuken/worker/inline_executor'
 require 'shoryuken/worker_registry'
 require 'shoryuken/default_worker_registry'
 require 'shoryuken/middleware/chain'
@@ -27,9 +34,11 @@ require 'shoryuken/polling/strict_priority'
 require 'shoryuken/manager'
 require 'shoryuken/launcher'
 require 'shoryuken/processor'
+require 'shoryuken/body_parser'
 require 'shoryuken/fetcher'
 require 'shoryuken/options'
 
+
 module Shoryuken
   extend SingleForwardable
 
@@ -42,6 +51,8 @@ module Shoryuken
     :ungrouped_queues,
     :worker_registry,
     :worker_registry=,
+    :worker_executor,
+    :worker_executor=,
     :polling_strategy,
     :start_callback,
     :start_callback=,

data/shoryuken.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'pry-byebug'
   spec.add_development_dependency 'dotenv'
 
-  spec.add_dependency 'aws-sdk-core', '> 2'
+  spec.add_dependency 'aws-sdk-core', '>= 2'
   spec.add_dependency 'concurrent-ruby'
   spec.add_dependency 'thor'
 end