shoryuken 5.0.5 → 6.1.1

data/gemfiles/rails_6_0.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 6.0"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/gemfiles/rails_6_1.gemfile

@@ -0,0 +1,21 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 6.1"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "pry-byebug", "3.9.0"
+  gem "rubocop"
+end
+
+gemspec path: "../"

data/gemfiles/rails_7_0.gemfile

@@ -0,0 +1,22 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+group :test do
+  gem "activejob", "~> 7.0"
+  gem "aws-sdk-core", "~> 3"
+  gem "aws-sdk-sqs"
+  gem "codeclimate-test-reporter", require: nil
+  gem "httparty"
+  gem "multi_xml"
+  gem "simplecov"
+end
+
+group :development do
+  gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
+  gem "rubocop"
+  gem "pry", ">= 0.14.2"
+  gem "pry-byebug", ">= 3.10.1"
+end
+
+gemspec path: "../"

data/lib/shoryuken/default_exception_handler.rb

@@ -0,0 +1,10 @@
+module Shoryuken
+  class DefaultExceptionHandler
+    extend Util
+
+    def self.call(exception, _queue, _sqs_msg)
+      logger.error { "Processor failed: #{exception.message}" }
+      logger.error { exception.backtrace.join("\n") } if exception.backtrace
+    end
+  end
+end

data/lib/shoryuken/environment_loader.rb

@@ -18,12 +18,12 @@ module Shoryuken
     end
 
     def setup_options
+      initialize_rails if load_rails?
       initialize_options
       initialize_logger
     end
 
     def load
-      load_rails if Shoryuken.options[:rails]
       prefix_active_job_queue_names
       parse_queues
       require_workers
@@ -55,7 +55,7 @@ module Shoryuken
       Shoryuken.logger.level = Logger::DEBUG if Shoryuken.options[:verbose]
     end
 
-    def load_rails
+    def initialize_rails
       # Adapted from: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/cli.rb
 
       require 'rails'
@@ -70,12 +70,22 @@ module Shoryuken
             ::Rails.application.config.eager_load = true
           end
         end
-        require 'shoryuken/extensions/active_job_adapter' if Shoryuken.active_job?
+        if Shoryuken.active_job?
+          require 'shoryuken/extensions/active_job_extensions'
+          require 'shoryuken/extensions/active_job_adapter'
+          require 'shoryuken/extensions/active_job_concurrent_send_adapter'
+        end
         require File.expand_path('config/environment.rb')
       end
     end
 
+    def load_rails?
+      options[:rails]
+    end
+
     def prefix_active_job_queue_name(queue_name, weight)
+      return [queue_name, weight] if queue_name.start_with?('https://', 'arn:')
+
       queue_name_prefix = ::ActiveJob::Base.queue_name_prefix
       queue_name_delimiter = ::ActiveJob::Base.queue_name_delimiter
 
@@ -153,9 +163,17 @@ module Shoryuken
 
       return if non_existent_queues.none?
 
+      # NOTE: HEREDOC's ~ operator removes indents, but is only available Ruby 2.3+
+      # See github PR: https://github.com/ruby-shoryuken/shoryuken/pull/691#issuecomment-1007653595
+      error_msg = <<-MSG.gsub(/^\s+/, '')
+        The specified queue(s) #{non_existent_queues.join(', ')} do not exist.
+        Try 'shoryuken sqs create QUEUE-NAME' for creating a queue with default settings.
+        It's also possible that you don't have permission to access the specified queues.
+      MSG
+
       fail(
         ArgumentError,
-        "The specified queue(s) #{non_existent_queues.join(', ')} do not exist.\nTry 'shoryuken sqs create QUEUE-NAME' for creating a queue with default settings"
+        error_msg
       )
     end
 

data/lib/shoryuken/extensions/active_job_adapter.rb

@@ -33,8 +33,12 @@ module ActiveJob
       def enqueue(job, options = {}) #:nodoc:
         register_worker!(job)
 
+        job.sqs_send_message_parameters.merge! options
+
         queue = Shoryuken::Client.queues(job.queue_name)
-        queue.send_message(message(queue, job, options))
+        send_message_params = message queue, job
+        job.sqs_send_message_parameters = send_message_params
+        queue.send_message send_message_params
       end
 
       def enqueue_at(job, timestamp) #:nodoc:
@@ -50,44 +54,50 @@ module ActiveJob
         delay
       end
 
-      def message(queue, job, options = {})
+      def message(queue, job)
         body = job.serialize
+        job_params = job.sqs_send_message_parameters
+
+        attributes = job_params[:message_attributes] || {}
 
-        msg = {}
+        msg = {
+          message_body: body,
+          message_attributes: attributes.merge(MESSAGE_ATTRIBUTES)
+        }
 
         if queue.fifo?
-          # See https://github.com/phstc/shoryuken/issues/457
-          msg[:message_deduplication_id] = Digest::SHA256.hexdigest(JSON.dump(body.except('job_id')))
+          # See https://github.com/ruby-shoryuken/shoryuken/issues/457 and
+          # https://github.com/ruby-shoryuken/shoryuken/pull/750#issuecomment-1781317929
+          msg[:message_deduplication_id] = Digest::SHA256.hexdigest(
+            JSON.dump(body.except('job_id', 'enqueued_at'))
+          )
         end
 
-        msg[:message_body] = body
-        msg[:message_attributes] = message_attributes
-
-        msg.merge(options)
+        msg.merge(job_params.except(:message_attributes))
       end
 
       def register_worker!(job)
         Shoryuken.register_worker(job.queue_name, JobWrapper)
       end
 
-      def message_attributes
-        @message_attributes ||= {
-          'shoryuken_class' => {
-            string_value: JobWrapper.to_s,
-            data_type: 'String'
-          }
-        }
-      end
-
       class JobWrapper #:nodoc:
         include Shoryuken::Worker
 
         shoryuken_options body_parser: :json, auto_delete: true
 
-        def perform(_sqs_msg, hash)
-          Base.execute hash
+        def perform(sqs_msg, hash)
+          receive_count = sqs_msg.attributes['ApproximateReceiveCount'].to_i
+          past_receives = receive_count - 1
+          Base.execute hash.merge({ 'executions' => past_receives })
         end
       end
+
+      MESSAGE_ATTRIBUTES = {
+        'shoryuken_class' => {
+          string_value: JobWrapper.to_s,
+          data_type: 'String'
+        }
+      }.freeze
     end
   end
 end

data/lib/shoryuken/extensions/active_job_extensions.rb

@@ -0,0 +1,38 @@
+module Shoryuken
+  module ActiveJobExtensions
+    # Adds an accessor for SQS SendMessage parameters on ActiveJob jobs
+    # (instances of ActiveJob::Base). Shoryuken ActiveJob queue adapters use
+    # these parameters when enqueueing jobs; other adapters can ignore them.
+    module SQSSendMessageParametersAccessor
+      extend ActiveSupport::Concern
+
+      included do
+        attr_accessor :sqs_send_message_parameters
+      end
+    end
+
+    # Initializes SQS SendMessage parameters on instances of ActiveJob::Base
+    # to the empty hash, and populates it whenever `#enqueue` is called, such
+    # as when using ActiveJob::Base.set.
+    module SQSSendMessageParametersSupport
+      def initialize(*arguments)
+        super(*arguments)
+        self.sqs_send_message_parameters = {}
+      end
+      ruby2_keywords(:initialize) if respond_to?(:ruby2_keywords, true)
+
+      def enqueue(options = {})
+        sqs_options = options.extract! :message_attributes,
+                                       :message_system_attributes,
+                                       :message_deduplication_id,
+                                       :message_group_id
+        sqs_send_message_parameters.merge! sqs_options
+
+        super
+      end
+    end
+  end
+end
+
+ActiveJob::Base.include Shoryuken::ActiveJobExtensions::SQSSendMessageParametersAccessor
+ActiveJob::Base.prepend Shoryuken::ActiveJobExtensions::SQSSendMessageParametersSupport

data/lib/shoryuken/launcher.rb

@@ -16,11 +16,13 @@ module Shoryuken
     def stop!
       initiate_stop
 
-      executor.shutdown
+      # Don't await here so the timeout below is not delayed
+      stop_new_dispatching
 
-      return if executor.wait_for_termination(Shoryuken.options[:timeout])
+      executor.shutdown
+      executor.kill unless executor.wait_for_termination(Shoryuken.options[:timeout])
 
-      executor.kill
+      fire_event(:stopped)
     end
 
     def stop
@@ -28,12 +30,32 @@ module Shoryuken
 
       initiate_stop
 
+      stop_new_dispatching
+      await_dispatching_in_progress
+
       executor.shutdown
       executor.wait_for_termination
+
+      fire_event(:stopped)
+    end
+
+    def healthy?
+      Shoryuken.groups.keys.all? do |group|
+        manager = @managers.find { |m| m.group == group }
+        manager && manager.running?
+      end
     end
 
     private
 
+    def stop_new_dispatching
+      @managers.each(&:stop_new_dispatching)
+    end
+
+    def await_dispatching_in_progress
+      @managers.each(&:await_dispatching_in_progress)
+    end
+
     def executor
       @_executor ||= Shoryuken.launcher_executor || Concurrent.global_io_executor
     end
@@ -71,6 +93,7 @@ module Shoryuken
     def create_managers
       Shoryuken.groups.map do |group, options|
         Shoryuken::Manager.new(
+          group,
           Shoryuken::Fetcher.new(group),
           Shoryuken.polling_strategy(group).new(options[:queues], Shoryuken.delay(group)),
           options[:concurrency],

data/lib/shoryuken/logging.rb

@@ -30,11 +30,11 @@ module Shoryuken
     end
 
     def self.logger
-      @logger || initialize_logger
+      @logger ||= initialize_logger
     end
 
     def self.logger=(log)
-      @logger = (log ? log : Logger.new('/dev/null'))
+      @logger = (log || Logger.new('/dev/null'))
     end
   end
 end

data/lib/shoryuken/manager.rb

@@ -6,27 +6,47 @@ module Shoryuken
     # See https://github.com/phstc/shoryuken/issues/348#issuecomment-292847028
     MIN_DISPATCH_INTERVAL = 0.1
 
-    def initialize(fetcher, polling_strategy, concurrency, executor)
-      @fetcher          = fetcher
-      @polling_strategy = polling_strategy
-      @max_processors   = concurrency
-      @busy_processors  = Concurrent::AtomicFixnum.new(0)
-      @executor         = executor
-      @running          = Concurrent::AtomicBoolean.new(true)
+    attr_reader :group
+
+    def initialize(group, fetcher, polling_strategy, concurrency, executor)
+      @group                      = group
+      @fetcher                    = fetcher
+      @polling_strategy           = polling_strategy
+      @max_processors             = concurrency
+      @busy_processors            = Concurrent::AtomicFixnum.new(0)
+      @executor                   = executor
+      @running                    = Concurrent::AtomicBoolean.new(true)
+      @stop_new_dispatching       = Concurrent::AtomicBoolean.new(false)
+      @dispatching_release_signal = ::Queue.new
     end
 
     def start
+      fire_utilization_update_event
       dispatch_loop
     end
 
-    private
+    def stop_new_dispatching
+      @stop_new_dispatching.make_true
+    end
+
+    def await_dispatching_in_progress
+      # There might still be a dispatching on-going, as the response from SQS could take some time
+      # We don't want to stop the process before processing incoming messages, as they would stay "in-flight" for some time on SQS
+      # We use a queue, as the dispatch_loop is running on another thread, and this is a efficient way of communicating between threads.
+      @dispatching_release_signal.pop
+    end
 
     def running?
       @running.true? && @executor.running?
     end
 
+    private
+
     def dispatch_loop
-      return unless running?
+      if @stop_new_dispatching.true? || !running?
+        @dispatching_release_signal << 1
+        return
+      end
 
       @executor.post { dispatch }
     end
@@ -57,8 +77,15 @@ module Shoryuken
       @max_processors - busy
     end
 
-    def processor_done
+    def processor_done(queue)
       @busy_processors.decrement
+      fire_utilization_update_event
+
+      client_queue = Shoryuken::Client.queues(queue)
+      return unless client_queue.fifo?
+      return unless @polling_strategy.respond_to?(:message_processed)
+
+      @polling_strategy.message_processed(queue)
     end
 
     def assign(queue_name, sqs_msg)
@@ -67,10 +94,12 @@ module Shoryuken
       logger.debug { "Assigning #{sqs_msg.message_id}" }
 
       @busy_processors.increment
+      fire_utilization_update_event
 
-      Concurrent::Promise.execute(
-        executor: @executor
-      ) { Processor.process(queue_name, sqs_msg) }.then { processor_done }.rescue { processor_done }
+      Concurrent::Promise
+        .execute(executor: @executor) { Processor.process(queue_name, sqs_msg) }
+        .then { processor_done(queue_name) }
+        .rescue { processor_done(queue_name) }
     end
 
     def dispatch_batch(queue)
@@ -81,7 +110,6 @@ module Shoryuken
 
     def dispatch_single_messages(queue)
       messages = @fetcher.fetch(queue, ready)
-
       @polling_strategy.messages_found(queue.name, messages.size)
       messages.each { |message| assign(queue.name, message) }
     end
@@ -108,5 +136,13 @@ module Shoryuken
 
       @running.make_false
     end
+
+    def fire_utilization_update_event
+      fire_event :utilization_update, false, {
+        group: @group,
+        max_processors: @max_processors,
+        busy_processors: busy
+      }
+    end
   end
 end

data/lib/shoryuken/message.rb

@@ -1,5 +1,16 @@
 module Shoryuken
   class Message
+    extend Forwardable
+
+    def_delegators(:data,
+                   :message_id,
+                   :receipt_handle,
+                   :md5_of_body,
+                   :body,
+                   :attributes,
+                   :md5_of_message_attributes,
+                   :message_attributes)
+
     attr_accessor :client, :queue_url, :queue_name, :data
 
     def initialize(client, queue, data)
@@ -29,33 +40,5 @@ module Shoryuken
         visibility_timeout: timeout
       )
     end
-
-    def message_id
-      data.message_id
-    end
-
-    def receipt_handle
-      data.receipt_handle
-    end
-
-    def md5_of_body
-      data.md5_of_body
-    end
-
-    def body
-      data.body
-    end
-
-    def attributes
-      data.attributes
-    end
-
-    def md5_of_message_attributes
-      data.md5_of_message_attributes
-    end
-
-    def message_attributes
-      data.message_attributes
-    end
   end
 end

data/lib/shoryuken/options.rb

@@ -9,20 +9,23 @@ module Shoryuken
       lifecycle_events: {
         startup: [],
         dispatch: [],
+        utilization_update: [],
         quiet: [],
-        shutdown: []
+        shutdown: [],
+        stopped: []
       }
     }.freeze
 
     attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout, :groups,
                   :launcher_executor,
-                  :start_callback, :stop_callback, :worker_executor, :worker_registry
+                  :start_callback, :stop_callback, :worker_executor, :worker_registry, :exception_handlers
     attr_writer :default_worker_options, :sqs_client
     attr_reader :sqs_client_receive_message_opts
 
     def initialize
       self.groups = {}
       self.worker_registry = DefaultWorkerRegistry.new
+      self.exception_handlers = [DefaultExceptionHandler]
       self.active_job_queue_name_prefixing = false
       self.worker_executor = Worker::DefaultExecutor
       self.cache_visibility_timeout = false
@@ -133,7 +136,7 @@ module Shoryuken
     end
 
     # Register a block to run at a point in the Shoryuken lifecycle.
-    # :startup, :quiet or :shutdown are valid events.
+    # :startup, :quiet, :shutdown or :stopped are valid events.
     #
     #   Shoryuken.configure_server do |config|
     #     config.on(:shutdown) do

data/lib/shoryuken/polling/base.rb

@@ -40,6 +40,8 @@ module Shoryuken
         fail NotImplementedError
       end
 
+      def message_processed(_queue); end
+
       def active_queues
         fail NotImplementedError
       end

data/lib/shoryuken/polling/strict_priority.rb

@@ -38,6 +38,13 @@ module Shoryuken
           .reverse
       end
 
+      def message_processed(queue)
+        if queue_paused?(queue)
+          logger.debug "Unpausing #{queue}"
+          @paused_until[queue] = Time.at 0
+        end
+      end
+
       private
 
       def next_active_queue
@@ -70,6 +77,7 @@ module Shoryuken
 
       def pause(queue)
         return unless delay > 0
+
         @paused_until[queue] = Time.now + delay
         logger.debug "Paused #{queue}"
       end

data/lib/shoryuken/polling/weighted_round_robin.rb

@@ -35,10 +35,18 @@ module Shoryuken
         unparse_queues(@queues)
       end
 
+      def message_processed(queue)
+        paused_queue = @paused_queues.find { |_time, name| name == queue }
+        return unless paused_queue
+
+        paused_queue[0] = Time.at 0
+      end
+
       private
 
       def pause(queue)
         return unless @queues.delete(queue)
+
         @paused_queues << [Time.now + delay, queue]
         logger.debug "Paused #{queue}"
       end
@@ -46,6 +54,7 @@ module Shoryuken
       def unpause_queues
         return if @paused_queues.empty?
         return if Time.now < @paused_queues.first[0]
+
         pause = @paused_queues.shift
         @queues << pause[1]
         logger.debug "Unpaused #{pause[1]}"

data/lib/shoryuken/processor.rb

@@ -22,8 +22,7 @@ module Shoryuken
         end
       end
     rescue Exception => ex
-      logger.error { "Processor failed: #{ex.message}" }
-      logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
+      Array(Shoryuken.exception_handlers).each { |handler| handler.call(ex, queue, sqs_msg) }
 
       raise
     end

data/lib/shoryuken/queue.rb

@@ -21,9 +21,10 @@ module Shoryuken
     end
 
     def delete_messages(options)
-      client.delete_message_batch(
+      failed_messages = client.delete_message_batch(
         options.merge(queue_url: url)
-      ).failed.any? do |failure|
+      ).failed || []
+      failed_messages.any? do |failure|
         logger.error do
           "Could not delete #{failure.id}, code: '#{failure.code}', message: '#{failure.message}', sender_fault: #{failure.sender_fault}"
         end
@@ -43,7 +44,8 @@ module Shoryuken
     end
 
     def receive_messages(options)
-      client.receive_message(options.merge(queue_url: url)).messages.map { |m| Message.new(client, self, m) }
+      messages = client.receive_message(options.merge(queue_url: url)).messages || []
+      messages.map { |m| Message.new(client, self, m) }
     end
 
     def fifo?

data/lib/shoryuken/runner.rb

@@ -30,9 +30,6 @@ module Shoryuken
 
       loader = EnvironmentLoader.setup_options(options)
 
-      # When cli args exist, override options in config file
-      Shoryuken.options.merge!(options)
-
       daemonize(Shoryuken.options)
       write_pid(Shoryuken.options)
 
@@ -55,6 +52,10 @@ module Shoryuken
       end
     end
 
+    def healthy?
+      (@launcher && @launcher.healthy?) || false
+    end
+
     private
 
     def initialize_concurrent_logger

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '5.0.5'.freeze
+  VERSION = '6.1.1'.freeze
 end