shoryuken 3.1.3 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 942ccdeaca0a94e766b82fa40da9ed091b703ea2
-  data.tar.gz: f94a089b56ed37b5db2752ccf418c667fae31b6d
+  metadata.gz: f74e5fbb599fe5bbe41bed5fd542c0c1fdcc2d35
+  data.tar.gz: 6099d42682b5e2a1797d7cdbdc7bf27b344b692c
 SHA512:
-  metadata.gz: 67c92d3af6c4063f42afe07a3f56c32b847bd1bc76d80f4f9f46cc2aa48d3e7bf82213f4d98fe229265ce9b1387a3649652863f019e73aeb0d809b1a03c5ca4d
-  data.tar.gz: 1215eca9e669975edb70fa43855122e6f25c6e54452771ba26435f7e8df37684be20516037e1419a3e7d93a14773231387f471cc2ff8a03eb838f1e52f2c144d
+  metadata.gz: f076e8f427f39ca92be4f27be819350d90270ce9e695d1173a6944b82d16680d11afd78a0075dd549e0a2c6031e6d5708c306cb245c4d7dfcc69d8a5e7151bc7
+  data.tar.gz: 2fd34c84e6260df3c35427ffb27d3d067fc1ea11469609d61e8eedd465e4516d75e052079b5195ede7c7917ad3c97bb39c5440bc5436ae3ab2581f3fb72bad31

data/.rubocop.yml

@@ -1,5 +1,5 @@
 AllCops:
-  TargetRubyVersion: 2.0
+  TargetRubyVersion: 2.1
 
 Style/SignalException:
   Enabled: false

data/.travis.yml

@@ -1,7 +1,8 @@
 language: ruby
+
 rvm:
   - 2.0.0
-  - 2.1.0
+  - 2.1.10
   - 2.2.0
   - 2.3.3
   - 2.4.1
@@ -11,9 +12,18 @@ notifications:
     on_success: change
     on_failure: always
 
-script: SPEC_ALL=true bundle exec rspec spec
+gemfile:
+  - Gemfile
+  - Gemfile.aws-sdk-core-v2
+
+env:
+  - SPEC_ALL=true
+
+script: bundle exec rspec spec
+
 before_install:
- - gem update bundler
+  - gem update bundler
+
 after_success:
   - bundle exec codeclimate-test-reporter
 

data/CHANGELOG.md

@@ -1,3 +1,71 @@
+## [v3.2.0] - 2018-01-03
+
+- Preserve parent worker class options
+ - [#451](https://github.com/phstc/shoryuken/pull/451)
+
+- Add -t (shutdown timeout) option to CL
+ - [#449](https://github.com/phstc/shoryuken/pull/449)
+
+- Support inline (Active Job like) for standard workers
+ - [#448](https://github.com/phstc/shoryuken/pull/448)
+
+## [v3.1.12] - 2017-09-25
+
+- Reduce fetch log verbosity
+ - [#436](https://github.com/phstc/shoryuken/pull/436)
+
+## [v3.1.11] - 2017-09-02
+
+- Auto retry (up to 3 times) fetch errors
+ - [#429](https://github.com/phstc/shoryuken/pull/429)
+
+## [v3.1.10] - 2017-09-02
+
+- Make Shoryuken compatible with AWS SDK 3 and 2
+ - [#433](https://github.com/phstc/shoryuken/pull/433)
+
+## [v3.1.9] - 2017-08-24
+
+- Add support for adding a middleware to the front of chain
+ - [#427](https://github.com/phstc/shoryuken/pull/427)
+
+- Add support for dispatch fire event
+ - [#426](https://github.com/phstc/shoryuken/pull/426)
+
+## [v3.1.8] - 2017-08-17
+
+- Make Polling strategy backward compatibility
+ - [#424](https://github.com/phstc/shoryuken/pull/424)
+
+## [v3.1.7] - 2017-07-31
+
+- Allow polling strategy per group
+ - [#417](https://github.com/phstc/shoryuken/pull/417)
+
+- Add support for creating FIFO queues
+ - [#419](https://github.com/phstc/shoryuken/pull/419)
+
+- Allow receive message options per queue
+ - [#420](https://github.com/phstc/shoryuken/pull/420)
+
+## [v3.1.6] - 2017-07-24
+
+- Fix issue with dispatch_loop and delays
+ - [#416](https://github.com/phstc/shoryuken/pull/416)
+
+## [v3.1.5] - 2017-07-23
+
+- Fix memory leak
+ - [#414](https://github.com/phstc/shoryuken/pull/414)
+
+- Fail fast on bad queue URLs
+ - [#413](https://github.com/phstc/shoryuken/pull/413)
+
+## [v3.1.4] - 2017-07-14
+
+- Require forwardable allowding to call `shoryuken` without `bundle exec`
+ - [#409](https://github.com/phstc/shoryuken/pull/409)
+
 ## [v3.1.3] - 2017-07-11
 
 - Add queue prefixing support for groups

data/Gemfile

@@ -4,9 +4,11 @@ source 'https://rubygems.org'
 gemspec
 
 group :test do
-  gem 'activejob'
+  gem 'activejob', '~> 4'
+  gem 'aws-sdk-core', '~> 3'
+  gem 'aws-sdk-sqs'
   gem 'codeclimate-test-reporter', require: nil
-  gem 'simplecov'
-  gem 'multi_xml'
   gem 'httparty'
+  gem 'multi_xml'
+  gem 'simplecov'
 end

data/Gemfile.aws-sdk-core-v2

@@ -0,0 +1,13 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in shoryuken.gemspec
+gemspec
+
+group :test do
+  gem 'activejob', '~> 4'
+  gem 'aws-sdk-core', '~> 2'
+  gem 'codeclimate-test-reporter', require: nil
+  gem 'httparty'
+  gem 'multi_xml'
+  gem 'simplecov'
+end

data/bin/cli/sqs.rb

@@ -96,8 +96,8 @@ module Shoryuken
       end
 
       desc 'ls [QUEUE-NAME-PREFIX]', 'Lists queues'
-      method_option :watch,          aliases: '-w',  type: :boolean,              desc: 'watch queues'
-      method_option :watch_interval,                 type: :numeric, default: 10, desc: 'watch interval'
+      method_option :watch,    aliases: '-w', type: :boolean, desc: 'watch queues'
+      method_option :interval, aliases: '-n', type: :numeric, default: 2, desc: 'watch interval in seconds'
       def ls(queue_name_prefix = '')
         trap('SIGINT', 'EXIT') # expect ctrl-c from loop
 
@@ -108,7 +108,7 @@ module Shoryuken
 
           break unless options[:watch]
 
-          sleep options[:watch_interval]
+          sleep options[:interval]
           puts
         end
       end
@@ -187,8 +187,12 @@ module Shoryuken
       end
 
       desc 'create QUEUE-NAME', 'Create a queue'
+      method_option :attributes, aliases: '-a', type: :hash, default: {}, desc: 'queue attributes'
       def create(queue_name)
-        queue_url = sqs.create_queue(queue_name: queue_name).queue_url
+        attributes = options[:attributes]
+        attributes['FifoQueue'] ||= 'true' if queue_name.end_with?('.fifo')
+
+        queue_url = sqs.create_queue(queue_name: queue_name, attributes: attributes).queue_url
 
         say "Queue #{queue_name} was successfully created. Queue URL #{queue_url}", :green
       end

data/bin/shoryuken

@@ -21,6 +21,7 @@ module Shoryuken
       method_option :daemon,      aliases: '-d', type: :boolean, desc: 'Daemonize process'
       method_option :queues,      aliases: '-q', type: :array,   desc: 'Queues to process with optional weights'
       method_option :require,     aliases: '-r', type: :string,  desc: 'Dir or path of the workers'
+      method_option :timeout,     aliases: '-t', type: :numeric, desc: 'Hard shutdown timeout'
       method_option :config,      aliases: '-C', type: :string,  desc: 'Path to config file'
       method_option :config_file,                type: :string,  desc: 'Path to config file (backwards compatibility)'
       method_option :rails,       aliases: '-R', type: :boolean, desc: 'Load Rails'
@@ -32,7 +33,7 @@ module Shoryuken
         opts = options.to_h.symbolize_keys
 
         if opts[:config_file]
-          say "[DEPRECATED] Please use --config instead of --config-file", :yellow
+          say '[DEPRECATED] Please use --config instead of --config-file', :yellow
         end
 
         opts[:config_file] = opts.delete(:config) if opts[:config]

data/lib/shoryuken/body_parser.rb

@@ -0,0 +1,27 @@
+module Shoryuken
+  class BodyParser
+    class << self
+      def parse(worker_class, sqs_msg)
+        body_parser = worker_class.get_shoryuken_options['body_parser']
+
+        case body_parser
+        when :json
+          JSON.parse(sqs_msg.body)
+        when Proc
+          body_parser.call(sqs_msg)
+        when :text, nil
+          sqs_msg.body
+        else
+          if body_parser.respond_to?(:parse)
+            # JSON.parse
+            body_parser.parse(sqs_msg.body)
+          elsif body_parser.respond_to?(:load)
+            # see https://github.com/phstc/shoryuken/pull/91
+            # JSON.load
+            body_parser.load(sqs_msg.body)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/fetcher.rb

@@ -4,34 +4,50 @@ module Shoryuken
 
     FETCH_LIMIT = 10
 
-    attr_reader :group
-
     def initialize(group)
       @group = group
     end
 
     def fetch(queue, limit)
-      started_at = Time.now
+      fetch_with_auto_retry(3) do
+        started_at = Time.now
 
-      logger.debug { "Looking for new messages in #{queue}" }
+        logger.debug { "Looking for new messages in #{queue}" }
 
-      sqs_msgs = Array(receive_messages(queue, [FETCH_LIMIT, limit].min))
+        sqs_msgs = Array(receive_messages(queue, [FETCH_LIMIT, limit].min))
 
-      logger.info { "Found #{sqs_msgs.size} messages for #{queue.name}" } unless sqs_msgs.empty?
-      logger.debug { "Fetcher for #{queue} completed in #{elapsed(started_at)} ms" }
+        logger.debug { "Found #{sqs_msgs.size} messages for #{queue.name}" } unless sqs_msgs.empty?
+        logger.debug { "Fetcher for #{queue} completed in #{elapsed(started_at)} ms" }
 
-      sqs_msgs
+        sqs_msgs
+      end
     end
 
     private
 
-    def receive_messages(queue, limit)
-      # AWS limits the batch size by 10
-      limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
+    def fetch_with_auto_retry(max_attempts, &block)
+      attempts = 0
+
+      begin
+        yield
+      rescue => ex
+        # Tries to auto retry connectivity errors
+        raise if attempts >= max_attempts
+
+        attempts += 1
+
+        logger.debug { "Retrying fetch attempt #{attempts} for #{ex.message}" }
+
+        sleep((1..5).to_a.sample)
+
+        retry
+      end
+    end
 
-      options = Shoryuken.sqs_client_receive_message_opts[group].to_h.dup
+    def receive_messages(queue, limit)
+      options = receive_options(queue)
 
-      options[:max_number_of_messages]  = limit
+      options[:max_number_of_messages]  = max_number_of_messages(limit, options)
       options[:message_attribute_names] = %w(All)
       options[:attribute_names]         = %w(All)
 
@@ -39,5 +55,16 @@ module Shoryuken
 
       Shoryuken::Client.queues(queue.name).receive_messages(options)
     end
+
+    def max_number_of_messages(limit, options)
+      [limit, FETCH_LIMIT, options[:max_number_of_messages]].compact.min
+    end
+
+    def receive_options(queue)
+      options = Shoryuken.sqs_client_receive_message_opts[queue.name]
+      options ||= Shoryuken.sqs_client_receive_message_opts[@group]
+
+      options.to_h.dup
+    end
   end
 end

data/lib/shoryuken/launcher.rb

@@ -3,8 +3,7 @@ module Shoryuken
     include Util
 
     def initialize
-      @managers    = create_managers
-      @shutdowning = Concurrent::AtomicBoolean.new(false)
+      @managers = create_managers
     end
 
     def start
@@ -41,24 +40,10 @@ module Shoryuken
 
     def start_managers
       @managers.each do |manager|
-        Concurrent::Promise.execute { manager.start }.rescue do |ex|
-          log_manager_failure(ex)
-          start_soft_shutdown
-        end
+        Concurrent::Future.execute { manager.start }
       end
     end
 
-    def start_soft_shutdown
-      Process.kill('USR1', Process.pid) if @shutdowning.make_true
-    end
-
-    def log_manager_failure(ex)
-      return unless ex
-
-      logger.error { "Manager failed: #{ex.message}" }
-      logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
-    end
-
     def initiate_stop
       logger.info { 'Shutting down' }
 

data/lib/shoryuken/manager.rb

@@ -12,6 +12,7 @@ module Shoryuken
       @max_processors   = concurrency
       @busy_processors  = Concurrent::AtomicFixnum.new(0)
       @executor         = executor
+      @running          = Concurrent::AtomicBoolean.new(true)
     end
 
     def start
@@ -21,15 +22,13 @@ module Shoryuken
     private
 
     def running?
-      @executor.running?
+      @running.true? && @executor.running?
     end
 
     def dispatch_loop
       return unless running?
 
-      Concurrent::Promise.execute(
-        executor: @executor
-      ) { dispatch }.then { dispatch_loop }.rescue { |ex| raise ex }
+      @executor.post { dispatch }
     end
 
     def dispatch
@@ -39,11 +38,15 @@ module Shoryuken
         return sleep(MIN_DISPATCH_INTERVAL)
       end
 
-      fire_event(:dispatch)
+      fire_event(:dispatch, false, queue_name: queue.name)
 
       logger.debug { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
 
       batched_queue?(queue) ? dispatch_batch(queue) : dispatch_single_messages(queue)
+    rescue => ex
+      handle_dispatch_error(ex)
+    ensure
+      dispatch_loop
     end
 
     def busy
@@ -96,5 +99,14 @@ module Shoryuken
 
       sqs_msgs
     end
+
+    def handle_dispatch_error(ex)
+      logger.error { "Manager failed: #{ex.message}" }
+      logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
+
+      Process.kill('USR1', Process.pid)
+
+      @running.make_false
+    end
   end
 end

data/lib/shoryuken/middleware/chain.rb

@@ -61,6 +61,10 @@ module Shoryuken
         entries << Entry.new(klass, *args) unless exists?(klass)
       end
 
+      def prepend(klass, *args)
+        entries.insert(0, Entry.new(klass, *args)) unless exists?(klass)
+      end
+
       def insert_before(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)

data/lib/shoryuken/middleware/server/auto_extend_visibility.rb

@@ -31,16 +31,13 @@ module Shoryuken
             Concurrent::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
               begin
                 logger.debug do
-                  "Extending message #{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id} " \
-                  "visibility timeout by #{queue_visibility_timeout}s."
+                  "Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
                 end
 
                 sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
               rescue => ex
                 logger.error do
-                  'Could not auto extend the message ' \
-                  "#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id} " \
-                  "visibility timeout. Error: #{ex.message}"
+                  "Could not auto extend the message #{queue}/#{sqs_msg.message_id} visibility timeout. Error: #{ex.message}"
                 end
               end
             end

data/lib/shoryuken/middleware/server/timing.rb

@@ -5,25 +5,23 @@ module Shoryuken
         include Util
 
         def call(worker, queue, sqs_msg, body)
-          Shoryuken::Logging.with_context("#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id}") do
-            begin
-              started_at = Time.now
+          begin
+            started_at = Time.now
 
-              logger.info { "started at #{started_at}" }
+            logger.info { "started at #{started_at}" }
 
-              yield
+            yield
 
-              total_time = elapsed(started_at)
+            total_time = elapsed(started_at)
 
-              if (total_time / 1000.0) > (timeout = Shoryuken::Client.queues(queue).visibility_timeout)
-                logger.warn { "exceeded the queue visibility timeout by #{total_time - (timeout * 1000)} ms" }
-              end
-
-              logger.info { "completed in: #{total_time} ms" }
-            rescue
-              logger.info { "failed in: #{elapsed(started_at)} ms" }
-              raise
+            if (total_time / 1000.0) > (timeout = Shoryuken::Client.queues(queue).visibility_timeout)
+              logger.warn { "exceeded the queue visibility timeout by #{total_time - (timeout * 1000)} ms" }
             end
+
+            logger.info { "completed in: #{total_time} ms" }
+          rescue
+            logger.info { "failed in: #{elapsed(started_at)} ms" }
+            raise
           end
         end
       end

data/lib/shoryuken/options.rb

@@ -21,6 +21,7 @@ module Shoryuken
     @@sqs_client_receive_message_opts = {}
     @@start_callback                  = nil
     @@stop_callback                   = nil
+    @@worker_executor                 = Worker::DefaultExecutor
 
     class << self
       def active_job?
@@ -56,8 +57,27 @@ module Shoryuken
         @@worker_registry = worker_registry
       end
 
+      def worker_executor
+        @@worker_executor
+      end
+
+      def worker_executor=(worker_executor)
+        @@worker_executor = worker_executor
+      end
+
       def polling_strategy(group)
-        options[group].to_h.fetch(:polling_strategy, Polling::WeightedRoundRobin)
+        strategy = (group == 'default' ? options : options[:groups].to_h[group]).to_h[:polling_strategy]
+
+        case strategy
+        when 'WeightedRoundRobin', nil # Default case
+          Polling::WeightedRoundRobin
+        when 'StrictPriority'
+          Polling::StrictPriority
+        when Class
+          strategy
+        else
+          raise ArgumentError, "#{strategy} is not a valid polling_strategy"
+        end
       end
 
       def start_callback

data/lib/shoryuken/processor.rb

@@ -16,8 +16,10 @@ module Shoryuken
     def process
       return logger.error { "No worker found for #{queue}" } unless worker
 
-      worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
-        worker.perform(sqs_msg, body)
+      Shoryuken::Logging.with_context("#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id}") do
+        worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
+          worker.perform(sqs_msg, body)
+        end
       end
     rescue Exception => ex
       logger.error { "Processor failed: #{ex.message}" }
@@ -41,25 +43,7 @@ module Shoryuken
     end
 
     def parse_body(sqs_msg)
-      body_parser = worker_class.get_shoryuken_options['body_parser']
-
-      case body_parser
-      when :json
-        JSON.parse(sqs_msg.body)
-      when Proc
-        body_parser.call(sqs_msg)
-      when :text, nil
-        sqs_msg.body
-      else
-        if body_parser.respond_to?(:parse)
-          # JSON.parse
-          body_parser.parse(sqs_msg.body)
-        elsif body_parser.respond_to?(:load)
-          # see https://github.com/phstc/shoryuken/pull/91
-          # JSON.load
-          body_parser.load(sqs_msg.body)
-        end
-      end
+      BodyParser.parse(worker_class, sqs_msg)
     end
   end
 end

data/lib/shoryuken/queue.rb

@@ -49,20 +49,27 @@ module Shoryuken
 
     private
 
-    def set_name(name)
+    def set_by_name(name)
       self.name = name
       self.url  = client.get_queue_url(queue_name: name).queue_url
-    rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue => ex
-      raise ex, "The specified queue #{name} does not exist."
     end
 
-    def set_url(url)
+    def set_by_url(url)
       self.name = url.split('/').last
       self.url  = url
     end
 
     def set_name_and_url(name_or_url)
-      name_or_url.start_with?('https://sqs.') ? set_url(name_or_url) : set_name(name_or_url)
+      if name_or_url.start_with?('https://sqs.')
+        set_by_url(name_or_url)
+
+        # anticipate the fifo? checker for validating the queue URL
+        return fifo?
+      end
+
+      set_by_name(name_or_url)
+    rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue => ex
+      raise ex, "The specified queue #{name_or_url} does not exist."
     end
 
     def queue_attributes

data/lib/shoryuken/runner.rb

@@ -118,10 +118,12 @@ module Shoryuken
     end
 
     def handle_signal(sig)
+      logger.debug "Got #{sig} signal"
+
       case sig
       when 'USR1' then execute_soft_shutdown
       when 'TTIN' then print_threads_backtrace
-      else
+      when 'TERM', 'INT'
         logger.info { "Received #{sig}, will shutdown down" }
 
         raise Interrupt

data/lib/shoryuken/util.rb

@@ -4,13 +4,13 @@ module Shoryuken
       Shoryuken.logger
     end
 
-    def fire_event(event, reverse = false)
+    def fire_event(event, reverse = false, event_options = {})
       logger.debug { "Firing '#{event}' lifecycle event" }
       arr = Shoryuken.options[:lifecycle_events][event]
       arr.reverse! if reverse
       arr.each do |block|
         begin
-          block.call
+          block.call(event_options)
         rescue => ex
           logger.warn(event: event)
           logger.warn "#{ex.class.name}: #{ex.message}"

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '3.1.3'.freeze
+  VERSION = '3.2.0'.freeze
 end

data/lib/shoryuken/worker/default_executor.rb

@@ -0,0 +1,33 @@
+module Shoryuken
+  module Worker
+    class DefaultExecutor
+      class << self
+        def perform_async(worker_class, body, options = {})
+          options[:message_attributes] ||= {}
+          options[:message_attributes]['shoryuken_class'] = {
+            string_value: worker_class.to_s,
+            data_type: 'String'
+          }
+
+          options[:message_body] = body
+
+          queue = options.delete(:queue) || worker_class.get_shoryuken_options['queue']
+
+          Shoryuken::Client.queues(queue).send_message(options)
+        end
+
+        def perform_in(worker_class, interval, body, options = {})
+          interval = interval.to_f
+          now = Time.now.to_f
+          ts = (interval < 1_000_000_000 ? (now + interval).to_f : interval)
+
+          delay = (ts - now).ceil
+
+          raise 'The maximum allowed delay is 15 minutes' if delay > 15 * 60
+
+          worker_class.perform_async(body, options.merge(delay_seconds: delay))
+        end
+      end
+    end
+  end
+end