shoryuken 3.1.0 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 779c3e9c7f708f0a6dccb3e8b0c9f7e2490b25e1
-  data.tar.gz: bf260a2793ff8c07ddfb670832ba5f12af8d85d9
+  metadata.gz: f74e5fbb599fe5bbe41bed5fd542c0c1fdcc2d35
+  data.tar.gz: 6099d42682b5e2a1797d7cdbdc7bf27b344b692c
 SHA512:
-  metadata.gz: 1f3e7a4b668c929daa1b5424b7a8ab7f028238b3f6dfe0e812351991bbc37923b58491760df6e4ed4921624cd4954761daeac257c6a540fd193d160d121044de
-  data.tar.gz: 02f2bf2e813cbe1624ae83d16936196ecf523fe8d3773f9b0c39813aa0822d3c71fd273ee01a63072c20d86bf05f4c1a775eda3fae41cec4bf55ea567f772aef
+  metadata.gz: f076e8f427f39ca92be4f27be819350d90270ce9e695d1173a6944b82d16680d11afd78a0075dd549e0a2c6031e6d5708c306cb245c4d7dfcc69d8a5e7151bc7
+  data.tar.gz: 2fd34c84e6260df3c35427ffb27d3d067fc1ea11469609d61e8eedd465e4516d75e052079b5195ede7c7917ad3c97bb39c5440bc5436ae3ab2581f3fb72bad31

data/.rubocop.yml

@@ -1,5 +1,5 @@
 AllCops:
-  TargetRubyVersion: 2.0
+  TargetRubyVersion: 2.1
 
 Style/SignalException:
   Enabled: false

data/.travis.yml

@@ -1,7 +1,8 @@
 language: ruby
+
 rvm:
   - 2.0.0
-  - 2.1.0
+  - 2.1.10
   - 2.2.0
   - 2.3.3
   - 2.4.1
@@ -11,9 +12,18 @@ notifications:
     on_success: change
     on_failure: always
 
-script: SPEC_ALL=true bundle exec rspec spec
+gemfile:
+  - Gemfile
+  - Gemfile.aws-sdk-core-v2
+
+env:
+  - SPEC_ALL=true
+
+script: bundle exec rspec spec
+
 before_install:
- - gem update bundler
+  - gem update bundler
+
 after_success:
   - bundle exec codeclimate-test-reporter
 

data/CHANGELOG.md

@@ -1,3 +1,92 @@
+## [v3.2.0] - 2018-01-03
+
+- Preserve parent worker class options
+ - [#451](https://github.com/phstc/shoryuken/pull/451)
+
+- Add -t (shutdown timeout) option to CL
+ - [#449](https://github.com/phstc/shoryuken/pull/449)
+
+- Support inline (Active Job like) for standard workers
+ - [#448](https://github.com/phstc/shoryuken/pull/448)
+
+## [v3.1.12] - 2017-09-25
+
+- Reduce fetch log verbosity
+ - [#436](https://github.com/phstc/shoryuken/pull/436)
+
+## [v3.1.11] - 2017-09-02
+
+- Auto retry (up to 3 times) fetch errors
+ - [#429](https://github.com/phstc/shoryuken/pull/429)
+
+## [v3.1.10] - 2017-09-02
+
+- Make Shoryuken compatible with AWS SDK 3 and 2
+ - [#433](https://github.com/phstc/shoryuken/pull/433)
+
+## [v3.1.9] - 2017-08-24
+
+- Add support for adding a middleware to the front of chain
+ - [#427](https://github.com/phstc/shoryuken/pull/427)
+
+- Add support for dispatch fire event
+ - [#426](https://github.com/phstc/shoryuken/pull/426)
+
+## [v3.1.8] - 2017-08-17
+
+- Make Polling strategy backward compatibility
+ - [#424](https://github.com/phstc/shoryuken/pull/424)
+
+## [v3.1.7] - 2017-07-31
+
+- Allow polling strategy per group
+ - [#417](https://github.com/phstc/shoryuken/pull/417)
+
+- Add support for creating FIFO queues
+ - [#419](https://github.com/phstc/shoryuken/pull/419)
+
+- Allow receive message options per queue
+ - [#420](https://github.com/phstc/shoryuken/pull/420)
+
+## [v3.1.6] - 2017-07-24
+
+- Fix issue with dispatch_loop and delays
+ - [#416](https://github.com/phstc/shoryuken/pull/416)
+
+## [v3.1.5] - 2017-07-23
+
+- Fix memory leak
+ - [#414](https://github.com/phstc/shoryuken/pull/414)
+
+- Fail fast on bad queue URLs
+ - [#413](https://github.com/phstc/shoryuken/pull/413)
+
+## [v3.1.4] - 2017-07-14
+
+- Require forwardable allowding to call `shoryuken` without `bundle exec`
+ - [#409](https://github.com/phstc/shoryuken/pull/409)
+
+## [v3.1.3] - 2017-07-11
+
+- Add queue prefixing support for groups
+ - [#405](https://github.com/phstc/shoryuken/pull/405)
+
+- Remove dead code
+ - [#402](https://github.com/phstc/shoryuken/pull/402)
+
+## [v3.1.2] - 2017-07-06
+
+- Fix stack level too deep on Ubuntu
+ - [#400](https://github.com/phstc/shoryuken/pull/400)
+
+## [v3.1.1] - 2017-07-05
+
+- Reduce log verbosity introduced in 3.1.0
+ - [#397](https://github.com/phstc/shoryuken/pull/397)
+
+- Try to prevent stack level too deep on Ubuntu
+ - [#396](https://github.com/phstc/shoryuken/pull/396)
+
 ## [v3.1.0] - 2017-07-02
 
 - Add shoryuken sqs delete command

data/Gemfile

@@ -4,8 +4,11 @@ source 'https://rubygems.org'
 gemspec
 
 group :test do
+  gem 'activejob', '~> 4'
+  gem 'aws-sdk-core', '~> 3'
+  gem 'aws-sdk-sqs'
   gem 'codeclimate-test-reporter', require: nil
-  gem 'simplecov'
-  gem 'multi_xml'
   gem 'httparty'
+  gem 'multi_xml'
+  gem 'simplecov'
 end

data/Gemfile.aws-sdk-core-v2

@@ -0,0 +1,13 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in shoryuken.gemspec
+gemspec
+
+group :test do
+  gem 'activejob', '~> 4'
+  gem 'aws-sdk-core', '~> 2'
+  gem 'codeclimate-test-reporter', require: nil
+  gem 'httparty'
+  gem 'multi_xml'
+  gem 'simplecov'
+end

data/bin/cli/sqs.rb

@@ -96,8 +96,8 @@ module Shoryuken
       end
 
       desc 'ls [QUEUE-NAME-PREFIX]', 'Lists queues'
-      method_option :watch,          aliases: '-w',  type: :boolean,              desc: 'watch queues'
-      method_option :watch_interval,                 type: :numeric, default: 10, desc: 'watch interval'
+      method_option :watch,    aliases: '-w', type: :boolean, desc: 'watch queues'
+      method_option :interval, aliases: '-n', type: :numeric, default: 2, desc: 'watch interval in seconds'
       def ls(queue_name_prefix = '')
         trap('SIGINT', 'EXIT') # expect ctrl-c from loop
 
@@ -108,7 +108,7 @@ module Shoryuken
 
           break unless options[:watch]
 
-          sleep options[:watch_interval]
+          sleep options[:interval]
           puts
         end
       end
@@ -187,8 +187,12 @@ module Shoryuken
       end
 
       desc 'create QUEUE-NAME', 'Create a queue'
+      method_option :attributes, aliases: '-a', type: :hash, default: {}, desc: 'queue attributes'
       def create(queue_name)
-        queue_url = sqs.create_queue(queue_name: queue_name).queue_url
+        attributes = options[:attributes]
+        attributes['FifoQueue'] ||= 'true' if queue_name.end_with?('.fifo')
+
+        queue_url = sqs.create_queue(queue_name: queue_name, attributes: attributes).queue_url
 
         say "Queue #{queue_name} was successfully created. Queue URL #{queue_url}", :green
       end

data/bin/shoryuken

@@ -21,6 +21,7 @@ module Shoryuken
       method_option :daemon,      aliases: '-d', type: :boolean, desc: 'Daemonize process'
       method_option :queues,      aliases: '-q', type: :array,   desc: 'Queues to process with optional weights'
       method_option :require,     aliases: '-r', type: :string,  desc: 'Dir or path of the workers'
+      method_option :timeout,     aliases: '-t', type: :numeric, desc: 'Hard shutdown timeout'
       method_option :config,      aliases: '-C', type: :string,  desc: 'Path to config file'
       method_option :config_file,                type: :string,  desc: 'Path to config file (backwards compatibility)'
       method_option :rails,       aliases: '-R', type: :boolean, desc: 'Load Rails'
@@ -32,7 +33,7 @@ module Shoryuken
         opts = options.to_h.symbolize_keys
 
         if opts[:config_file]
-          say "[DEPRECATED] Please use --config instead of --config-file", :yellow
+          say '[DEPRECATED] Please use --config instead of --config-file', :yellow
         end
 
         opts[:config_file] = opts.delete(:config) if opts[:config]

data/lib/shoryuken/body_parser.rb

@@ -0,0 +1,27 @@
+module Shoryuken
+  class BodyParser
+    class << self
+      def parse(worker_class, sqs_msg)
+        body_parser = worker_class.get_shoryuken_options['body_parser']
+
+        case body_parser
+        when :json
+          JSON.parse(sqs_msg.body)
+        when Proc
+          body_parser.call(sqs_msg)
+        when :text, nil
+          sqs_msg.body
+        else
+          if body_parser.respond_to?(:parse)
+            # JSON.parse
+            body_parser.parse(sqs_msg.body)
+          elsif body_parser.respond_to?(:load)
+            # see https://github.com/phstc/shoryuken/pull/91
+            # JSON.load
+            body_parser.load(sqs_msg.body)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/environment_loader.rb

@@ -44,7 +44,7 @@ module Shoryuken
 
       fail ArgumentError, "The supplied config file #{path} does not exist" unless File.exist?(path)
 
-      if result = YAML.load(ERB.new(IO.read(path)).result)
+      if (result = YAML.load(ERB.new(IO.read(path)).result))
         result.deep_symbolize_keys
       else
         {}
@@ -69,7 +69,7 @@ module Shoryuken
         ::Rails::Application.initializer 'shoryuken.eager_load' do
           ::Rails.application.config.eager_load = true
         end
-        require 'shoryuken/extensions/active_job_adapter' if defined?(::ActiveJob)
+        require 'shoryuken/extensions/active_job_adapter' if Shoryuken.active_job?
         require File.expand_path('config/environment.rb')
       end
     end
@@ -85,22 +85,36 @@ module Shoryuken
       end
     end
 
-    def prefix_active_job_queue_names
-      return unless defined? ::ActiveJob
-      return unless Shoryuken.active_job_queue_name_prefixing
-
+    def prefix_active_job_queue_name(queue_name, weight)
       queue_name_prefix = ::ActiveJob::Base.queue_name_prefix
       queue_name_delimiter = ::ActiveJob::Base.queue_name_delimiter
 
       # See https://github.com/rails/rails/blob/master/activejob/lib/active_job/queue_name.rb#L27
+      name_parts = [queue_name_prefix.presence, queue_name]
+      prefixed_queue_name = name_parts.compact.join(queue_name_delimiter)
+      [prefixed_queue_name, weight]
+    end
+
+    def prefix_active_job_queue_names
+      return unless Shoryuken.active_job?
+      return unless Shoryuken.active_job_queue_name_prefixing
+
       Shoryuken.options[:queues].to_a.map! do |queue_name, weight|
-        name_parts = [queue_name_prefix.presence, queue_name]
-        prefixed_queue_name = name_parts.compact.join(queue_name_delimiter)
-        [prefixed_queue_name, weight]
+        prefix_active_job_queue_name(queue_name, weight)
+      end
+
+      Shoryuken.options[:groups].to_a.map! do |group, options|
+        if options[:queues]
+          options[:queues].map! do |queue_name, weight|
+            prefix_active_job_queue_name(queue_name, weight)
+          end
+        end
+
+        [group, options]
       end
     end
 
-    def parse_queue(queue, weight = nil, group)
+    def parse_queue(queue, weight, group)
       Shoryuken.add_queue(queue, [weight.to_i, 1].max, group)
     end
 
@@ -155,7 +169,7 @@ module Shoryuken
     end
 
     def validate_workers
-      return if defined?(::ActiveJob)
+      return if Shoryuken.active_job?
 
       all_queues = Shoryuken.ungrouped_queues
       queues_with_workers = Shoryuken.worker_registry.queues

data/lib/shoryuken/fetcher.rb

@@ -4,34 +4,50 @@ module Shoryuken
 
     FETCH_LIMIT = 10
 
-    attr_reader :group
-
     def initialize(group)
       @group = group
     end
 
     def fetch(queue, limit)
-      started_at = Time.now
+      fetch_with_auto_retry(3) do
+        started_at = Time.now
 
-      logger.debug { "Looking for new messages in #{queue}" }
+        logger.debug { "Looking for new messages in #{queue}" }
 
-      sqs_msgs = Array(receive_messages(queue, [FETCH_LIMIT, limit].min))
+        sqs_msgs = Array(receive_messages(queue, [FETCH_LIMIT, limit].min))
 
-      logger.info { "Found #{sqs_msgs.size} messages for #{queue.name}" } unless sqs_msgs.empty?
-      logger.debug { "Fetcher for #{queue} completed in #{elapsed(started_at)} ms" }
+        logger.debug { "Found #{sqs_msgs.size} messages for #{queue.name}" } unless sqs_msgs.empty?
+        logger.debug { "Fetcher for #{queue} completed in #{elapsed(started_at)} ms" }
 
-      sqs_msgs
+        sqs_msgs
+      end
     end
 
     private
 
-    def receive_messages(queue, limit)
-      # AWS limits the batch size by 10
-      limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
+    def fetch_with_auto_retry(max_attempts, &block)
+      attempts = 0
+
+      begin
+        yield
+      rescue => ex
+        # Tries to auto retry connectivity errors
+        raise if attempts >= max_attempts
+
+        attempts += 1
+
+        logger.debug { "Retrying fetch attempt #{attempts} for #{ex.message}" }
+
+        sleep((1..5).to_a.sample)
+
+        retry
+      end
+    end
 
-      options = Shoryuken.sqs_client_receive_message_opts[group].to_h.dup
+    def receive_messages(queue, limit)
+      options = receive_options(queue)
 
-      options[:max_number_of_messages]  = limit
+      options[:max_number_of_messages]  = max_number_of_messages(limit, options)
       options[:message_attribute_names] = %w(All)
       options[:attribute_names]         = %w(All)
 
@@ -39,5 +55,16 @@ module Shoryuken
 
       Shoryuken::Client.queues(queue.name).receive_messages(options)
     end
+
+    def max_number_of_messages(limit, options)
+      [limit, FETCH_LIMIT, options[:max_number_of_messages]].compact.min
+    end
+
+    def receive_options(queue)
+      options = Shoryuken.sqs_client_receive_message_opts[queue.name]
+      options ||= Shoryuken.sqs_client_receive_message_opts[@group]
+
+      options.to_h.dup
+    end
   end
 end

data/lib/shoryuken/launcher.rb

@@ -3,8 +3,7 @@ module Shoryuken
     include Util
 
     def initialize
-      @managers    = create_managers
-      @shutdowning = Concurrent::AtomicBoolean.new(false)
+      @managers = create_managers
     end
 
     def start
@@ -41,29 +40,13 @@ module Shoryuken
 
     def start_managers
       @managers.each do |manager|
-        Concurrent::Promise.execute { manager.start }.rescue do |ex|
-          log_manager_failure(ex)
-          start_soft_shutdown
-        end
+        Concurrent::Future.execute { manager.start }
       end
     end
 
-    def start_soft_shutdown
-      Process.kill('USR1', Process.pid) if @shutdowning.make_true
-    end
-
-    def log_manager_failure(ex)
-      return unless ex
-
-      logger.error { "Manager failed: #{ex.message}" }
-      logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
-    end
-
     def initiate_stop
       logger.info { 'Shutting down' }
 
-      @managers.each(&:stop)
-
       stop_callback
     end
 
@@ -90,7 +73,8 @@ module Shoryuken
         Shoryuken::Manager.new(
           Shoryuken::Fetcher.new(group),
           Shoryuken.polling_strategy(group).new(options[:queues]),
-          options[:concurrency]
+          options[:concurrency],
+          executor
         )
       end
     end

data/lib/shoryuken/logging.rb

@@ -3,7 +3,6 @@ require 'logger'
 
 module Shoryuken
   module Logging
-
     class Pretty < Logger::Formatter
       # Provide a call() method that returns the formatted message.
       def call(severity, time, program_name, message)
@@ -37,9 +36,5 @@ module Shoryuken
     def self.logger=(log)
       @logger = (log ? log : Logger.new('/dev/null'))
     end
-
-    def logger
-      shoryuken::Logging.logger
-    end
   end
 end

data/lib/shoryuken/manager.rb

@@ -6,47 +6,47 @@ module Shoryuken
     # See https://github.com/phstc/shoryuken/issues/348#issuecomment-292847028
     MIN_DISPATCH_INTERVAL = 0.1
 
-    def initialize(fetcher, polling_strategy, concurrency)
+    def initialize(fetcher, polling_strategy, concurrency, executor)
       @fetcher          = fetcher
       @polling_strategy = polling_strategy
       @max_processors   = concurrency
       @busy_processors  = Concurrent::AtomicFixnum.new(0)
-      @done             = Concurrent::AtomicBoolean.new(false)
+      @executor         = executor
+      @running          = Concurrent::AtomicBoolean.new(true)
     end
 
     def start
-      dispatch
+      dispatch_loop
     end
 
-    def stop
-      @done.make_true
+    private
+
+    def running?
+      @running.true? && @executor.running?
     end
 
-    private
+    def dispatch_loop
+      return unless running?
 
-    def stopped?
-      @done.true? || !Concurrent.global_io_executor.running?
+      @executor.post { dispatch }
     end
 
     def dispatch
-      return if stopped?
+      return unless running?
 
-      if !ready.positive? || (queue = @polling_strategy.next_queue).nil?
-        return dispatch_later
+      if ready <= 0 || (queue = @polling_strategy.next_queue).nil?
+        return sleep(MIN_DISPATCH_INTERVAL)
       end
 
-      fire_event(:dispatch)
+      fire_event(:dispatch, false, queue_name: queue.name)
 
-      logger.info { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
+      logger.debug { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
 
       batched_queue?(queue) ? dispatch_batch(queue) : dispatch_single_messages(queue)
-
-      dispatch
-    end
-
-    def dispatch_later
-      sleep(MIN_DISPATCH_INTERVAL)
-      dispatch
+    rescue => ex
+      handle_dispatch_error(ex)
+    ensure
+      dispatch_loop
     end
 
     def busy
@@ -62,15 +62,15 @@ module Shoryuken
     end
 
     def assign(queue_name, sqs_msg)
-      return if stopped?
+      return unless running?
 
       logger.debug { "Assigning #{sqs_msg.message_id}" }
 
       @busy_processors.increment
 
-      Concurrent::Promise.execute {
-        Processor.new(queue_name, sqs_msg).process
-      }.then { processor_done }.rescue { processor_done }
+      Concurrent::Promise.execute(
+        executor: @executor
+      ) { Processor.process(queue_name, sqs_msg) }.then { processor_done }.rescue { processor_done }
     end
 
     def dispatch_batch(queue)
@@ -99,5 +99,14 @@ module Shoryuken
 
       sqs_msgs
     end
+
+    def handle_dispatch_error(ex)
+      logger.error { "Manager failed: #{ex.message}" }
+      logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
+
+      Process.kill('USR1', Process.pid)
+
+      @running.make_false
+    end
   end
 end

data/lib/shoryuken/middleware/chain.rb

@@ -61,6 +61,10 @@ module Shoryuken
         entries << Entry.new(klass, *args) unless exists?(klass)
       end
 
+      def prepend(klass, *args)
+        entries.insert(0, Entry.new(klass, *args)) unless exists?(klass)
+      end
+
       def insert_before(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)

data/lib/shoryuken/middleware/server/auto_extend_visibility.rb

@@ -31,16 +31,13 @@ module Shoryuken
             Concurrent::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
               begin
                 logger.debug do
-                  "Extending message #{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id} " \
-                  "visibility timeout by #{queue_visibility_timeout}s."
+                  "Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
                 end
 
                 sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
               rescue => ex
                 logger.error do
-                  'Could not auto extend the message ' \
-                  "#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id} " \
-                  "visibility timeout. Error: #{ex.message}"
+                  "Could not auto extend the message #{queue}/#{sqs_msg.message_id} visibility timeout. Error: #{ex.message}"
                 end
               end
             end

data/lib/shoryuken/middleware/server/timing.rb

@@ -5,25 +5,23 @@ module Shoryuken
         include Util
 
         def call(worker, queue, sqs_msg, body)
-          Shoryuken::Logging.with_context("#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id}") do
-            begin
-              started_at = Time.now
+          begin
+            started_at = Time.now
 
-              logger.info { "started at #{started_at}" }
+            logger.info { "started at #{started_at}" }
 
-              yield
+            yield
 
-              total_time = elapsed(started_at)
+            total_time = elapsed(started_at)
 
-              if (total_time / 1000.0) > (timeout = Shoryuken::Client.queues(queue).visibility_timeout)
-                logger.warn { "exceeded the queue visibility timeout by #{total_time - (timeout * 1000)} ms" }
-              end
-
-              logger.info { "completed in: #{total_time} ms" }
-            rescue
-              logger.info { "failed in: #{elapsed(started_at)} ms" }
-              raise
+            if (total_time / 1000.0) > (timeout = Shoryuken::Client.queues(queue).visibility_timeout)
+              logger.warn { "exceeded the queue visibility timeout by #{total_time - (timeout * 1000)} ms" }
             end
+
+            logger.info { "completed in: #{total_time} ms" }
+          rescue
+            logger.info { "failed in: #{elapsed(started_at)} ms" }
+            raise
           end
         end
       end