shoryuken 3.0.6 → 4.0.0

data/lib/shoryuken/extensions/active_job_adapter.rb

@@ -30,30 +30,40 @@ module ActiveJob
         end
       end
 
-      def enqueue(job) #:nodoc:
+      def enqueue(job, options = {}) #:nodoc:
         register_worker!(job)
 
         queue = Shoryuken::Client.queues(job.queue_name)
-        queue.send_message(message(job))
+        queue.send_message(message(queue, job, options))
       end
 
       def enqueue_at(job, timestamp) #:nodoc:
-        register_worker!(job)
+        enqueue(job, delay_seconds: calculate_delay(timestamp))
+      end
+
+      private
 
+      def calculate_delay(timestamp)
         delay = (timestamp - Time.current.to_f).round
         raise 'The maximum allowed delay is 15 minutes' if delay > 15.minutes
 
-        queue = Shoryuken::Client.queues(job.queue_name)
-        queue.send_message(message(job, delay_seconds: delay))
+        delay
       end
 
-      private
-
-      def message(job, options = {})
+      def message(queue, job, options = {})
         body = job.serialize
 
-        { message_body: body,
-          message_attributes: message_attributes }.merge(options)
+        msg = {}
+
+        if queue.fifo?
+          # See https://github.com/phstc/shoryuken/issues/457
+          msg[:message_deduplication_id] = Digest::SHA256.hexdigest(JSON.dump(body.except('job_id')))
+        end
+
+        msg[:message_body] = body
+        msg[:message_attributes] = message_attributes
+
+        msg.merge(options)
       end
 
       def register_worker!(job)
@@ -74,7 +84,7 @@ module ActiveJob
 
         shoryuken_options body_parser: :json, auto_delete: true
 
-        def perform(sqs_msg, hash)
+        def perform(_sqs_msg, hash)
           Base.execute hash
         end
       end

data/lib/shoryuken/fetcher.rb

@@ -4,39 +4,78 @@ module Shoryuken
 
     FETCH_LIMIT = 10
 
-    def fetch(queue, available_processors)
-      started_at = Time.now
+    def initialize(group)
+      @group = group
+    end
 
-      logger.debug { "Looking for new messages in '#{queue}'" }
+    def fetch(queue, limit)
+      fetch_with_auto_retry(3) do
+        started_at = Time.now
 
-      begin
-        limit = available_processors > FETCH_LIMIT ? FETCH_LIMIT : available_processors
+        logger.debug { "Looking for new messages in #{queue}" }
+
+        sqs_msgs = Array(receive_messages(queue, [FETCH_LIMIT, limit].min))
+
+        logger.debug { "Found #{sqs_msgs.size} messages for #{queue.name}" } unless sqs_msgs.empty?
+        logger.debug { "Fetcher for #{queue} completed in #{elapsed(started_at)} ms" }
 
-        sqs_msgs = Array(receive_messages(queue, limit))
-        logger.info { "Found #{sqs_msgs.size} messages for '#{queue.name}'" } unless sqs_msgs.empty?
-        logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
         sqs_msgs
-      rescue => ex
-        logger.error { "Error fetching message: #{ex}" }
-        logger.error { ex.backtrace.first }
-        []
       end
     end
 
     private
 
+    def fetch_with_auto_retry(max_attempts)
+      attempts = 0
+
+      begin
+        yield
+      rescue => ex
+        # Tries to auto retry connectivity errors
+        raise if attempts >= max_attempts
+
+        attempts += 1
+
+        logger.debug { "Retrying fetch attempt #{attempts} for #{ex.message}" }
+
+        sleep((1..5).to_a.sample)
+
+        retry
+      end
+    end
+
     def receive_messages(queue, limit)
-      # AWS limits the batch size by 10
-      limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
+      options = receive_options(queue)
+
+      shoryuken_queue = Shoryuken::Client.queues(queue.name)
 
-      options = Shoryuken.sqs_client_receive_message_opts.to_h.dup
-      options[:max_number_of_messages] = limit
-      options[:message_attribute_names] = %w(All)
-      options[:attribute_names] = %w(All)
+      # For FIFO queues we want to make sure we process one message per group at the time
+      # if we set max_number_of_messages greater than 1,
+      # SQS may return more than one message for the same message group
+      # since Shoryuken uses threads, it will try to process more than one at once
+      # > The message group ID is the tag that specifies that a message belongs to a specific message group.
+      # > Messages that belong to the same message group are always processed one by one,
+      # > in a strict order relative to the message group
+      # > (however, messages that belong to different message groups might be processed out of order).
+      # > https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagegroupid-property.html
+      options[:max_number_of_messages]  = shoryuken_queue.fifo? ? 1 : max_number_of_messages(limit, options)
+      options[:message_attribute_names] = %w[All]
+      options[:attribute_names]         = %w[All]
 
       options.merge!(queue.options)
 
-      Shoryuken::Client.queues(queue.name).receive_messages(options)
+      shoryuken_queue.receive_messages(options)
+    end
+
+    def max_number_of_messages(limit, options)
+      [limit, FETCH_LIMIT, options[:max_number_of_messages]].compact.min
+    end
+
+    def receive_options(queue)
+      options = Shoryuken.sqs_client_receive_message_opts[queue.name]
+      options ||= Shoryuken.sqs_client_receive_message_opts[@group]
+
+      options.to_h.dup
     end
   end
 end

data/lib/shoryuken/launcher.rb

@@ -3,17 +3,80 @@ module Shoryuken
     include Util
 
     def initialize
-      @manager = Shoryuken::Manager.new(Shoryuken::Fetcher.new,
-                                        Shoryuken.options[:polling_strategy].new(Shoryuken.queues))
+      @managers = create_managers
     end
 
-    def stop(options = {})
-      @manager.stop(shutdown: !options[:shutdown].nil?,
-                    timeout: Shoryuken.options[:timeout])
+    def start
+      logger.info { 'Starting' }
+
+      start_callback
+      start_managers
+    end
+
+    def stop!
+      initiate_stop
+
+      executor.shutdown
+
+      return if executor.wait_for_termination(Shoryuken.options[:timeout])
+
+      executor.kill
+    end
+
+    def stop
+      fire_event(:quiet, true)
+
+      initiate_stop
+
+      executor.shutdown
+      executor.wait_for_termination
+    end
+
+    private
+
+    def executor
+      @_executor ||= Shoryuken.launcher_executor || Concurrent.global_io_executor
+    end
+
+    def start_managers
+      @managers.each do |manager|
+        Concurrent::Future.execute { manager.start }
+      end
+    end
+
+    def initiate_stop
+      logger.info { 'Shutting down' }
+
+      stop_callback
+    end
+
+    def start_callback
+      if (callback = Shoryuken.start_callback)
+        logger.debug { 'Calling start_callback' }
+        callback.call
+      end
+
+      fire_event(:startup)
+    end
+
+    def stop_callback
+      if (callback = Shoryuken.stop_callback)
+        logger.debug { 'Calling stop_callback' }
+        callback.call
+      end
+
+      fire_event(:shutdown, true)
     end
 
-    def run
-      @manager.start
+    def create_managers
+      Shoryuken.groups.map do |group, options|
+        Shoryuken::Manager.new(
+          Shoryuken::Fetcher.new(group),
+          Shoryuken.polling_strategy(group).new(options[:queues]),
+          options[:concurrency],
+          executor
+        )
+      end
     end
   end
 end

data/lib/shoryuken/logging.rb

@@ -3,10 +3,9 @@ require 'logger'
 
 module Shoryuken
   module Logging
-
     class Pretty < Logger::Formatter
       # Provide a call() method that returns the formatted message.
-      def call(severity, time, program_name, message)
+      def call(severity, time, _program_name, message)
         "#{time.utc.iso8601} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
       end
 
@@ -37,9 +36,5 @@ module Shoryuken
     def self.logger=(log)
       @logger = (log ? log : Logger.new('/dev/null'))
     end
-
-    def logger
-      shoryuken::Logging.logger
-    end
   end
 end

data/lib/shoryuken/manager.rb

@@ -6,103 +6,82 @@ module Shoryuken
     # See https://github.com/phstc/shoryuken/issues/348#issuecomment-292847028
     MIN_DISPATCH_INTERVAL = 0.1
 
-    def initialize(fetcher, polling_strategy)
-      @count = Shoryuken.options.fetch(:concurrency, 25)
-
-      raise(ArgumentError, "Concurrency value #{@count} is invalid, it needs to be a positive number") unless @count > 0
-
-      @queues = Shoryuken.queues.dup.uniq
-
-      @done = Concurrent::AtomicBoolean.new(false)
-
-      @fetcher = fetcher
+    def initialize(fetcher, polling_strategy, concurrency, executor)
+      @fetcher          = fetcher
       @polling_strategy = polling_strategy
-
-      @pool = Concurrent::FixedThreadPool.new(@count, max_queue: @count)
-      @dispatcher_executor = Concurrent::SingleThreadExecutor.new
+      @max_processors   = concurrency
+      @busy_processors  = Concurrent::AtomicFixnum.new(0)
+      @executor         = executor
+      @running          = Concurrent::AtomicBoolean.new(true)
     end
 
     def start
-      logger.info { 'Starting' }
-
-      dispatch_async
+      dispatch_loop
     end
 
-    def stop(options = {})
-      @done.make_true
-
-      if (callback = Shoryuken.stop_callback)
-        logger.info { 'Calling Shoryuken.on_stop block' }
-        callback.call
-      end
-
-      fire_event(:shutdown, true)
-
-      logger.info { 'Shutting down workers' }
-
-      @dispatcher_executor.kill
+    private
 
-      if options[:shutdown]
-        hard_shutdown_in(options[:timeout])
-      else
-        soft_shutdown
-      end
+    def running?
+      @running.true? && @executor.running?
     end
 
-    def processor_failed(ex)
-      logger.error ex
-      logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
-    end
+    def dispatch_loop
+      return unless running?
 
-    def processor_done(queue)
-      logger.debug { "Process done for '#{queue}'" }
+      @executor.post { dispatch }
     end
 
-    private
+    def dispatch
+      return unless running?
 
-    def dispatch_async
-      @dispatcher_executor.post(&method(:dispatch_now))
-    end
-
-    def dispatch_now
-      return if @done.true?
+      if ready <= 0 || (queue = @polling_strategy.next_queue).nil?
+        return sleep(MIN_DISPATCH_INTERVAL)
+      end
 
-      begin
-        if ready.zero? || (queue = @polling_strategy.next_queue).nil?
-          sleep MIN_DISPATCH_INTERVAL
-          return
-        end
+      fire_event(:dispatch, false, queue_name: queue.name)
 
-        logger.debug { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
+      logger.debug { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
 
-        batched_queue?(queue) ? dispatch_batch(queue) : dispatch_single_messages(queue)
-      ensure
-        dispatch_async
-      end
+      batched_queue?(queue) ? dispatch_batch(queue) : dispatch_single_messages(queue)
+    rescue => ex
+      handle_dispatch_error(ex)
+    ensure
+      dispatch_loop
     end
 
     def busy
-      @count - ready
+      @busy_processors.value
     end
 
     def ready
-      @pool.remaining_capacity
+      @max_processors - busy
+    end
+
+    def processor_done
+      @busy_processors.decrement
     end
 
-    def assign(queue, sqs_msg)
+    def assign(queue_name, sqs_msg)
+      return unless running?
+
       logger.debug { "Assigning #{sqs_msg.message_id}" }
 
-      @pool.post { Processor.new(self).process(queue, sqs_msg) }
+      @busy_processors.increment
+
+      Concurrent::Promise.execute(
+        executor: @executor
+      ) { Processor.process(queue_name, sqs_msg) }.then { processor_done }.rescue { processor_done }
     end
 
     def dispatch_batch(queue)
-      batch = @fetcher.fetch(queue, BATCH_LIMIT)
+      return if (batch = @fetcher.fetch(queue, BATCH_LIMIT)).none?
       @polling_strategy.messages_found(queue.name, batch.size)
       assign(queue.name, patch_batch!(batch))
     end
 
     def dispatch_single_messages(queue)
       messages = @fetcher.fetch(queue, ready)
+
       @polling_strategy.messages_found(queue.name, messages.size)
       messages.each { |message| assign(queue.name, message) }
     end
@@ -111,24 +90,6 @@ module Shoryuken
       Shoryuken.worker_registry.batch_receive_messages?(queue.name)
     end
 
-    def soft_shutdown
-      @pool.shutdown
-      @pool.wait_for_termination
-    end
-
-    def hard_shutdown_in(delay)
-      if busy > 0
-        logger.info { "Pausing up to #{delay} seconds to allow workers to finish..." }
-      end
-
-      @pool.shutdown
-
-      return if @pool.wait_for_termination(delay)
-
-      logger.info { "Hard shutting down #{busy} busy workers" }
-      @pool.kill
-    end
-
     def patch_batch!(sqs_msgs)
       sqs_msgs.instance_eval do
         def message_id
@@ -138,5 +99,14 @@ module Shoryuken
 
       sqs_msgs
     end
+
+    def handle_dispatch_error(ex)
+      logger.error { "Manager failed: #{ex.message}" }
+      logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
+
+      Process.kill('USR1', Process.pid)
+
+      @running.make_false
+    end
   end
 end

data/lib/shoryuken/middleware/chain.rb

@@ -61,6 +61,10 @@ module Shoryuken
         entries << Entry.new(klass, *args) unless exists?(klass)
       end
 
+      def prepend(klass, *args)
+        entries.insert(0, Entry.new(klass, *args)) unless exists?(klass)
+      end
+
       def insert_before(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)

data/lib/shoryuken/middleware/server/active_record.rb

@@ -2,7 +2,7 @@ module Shoryuken
   module Middleware
     module Server
       class ActiveRecord
-        def call(*args)
+        def call(*_args)
           yield
         ensure
           ::ActiveRecord::Base.clear_active_connections!

data/lib/shoryuken/middleware/server/auto_delete.rb

@@ -2,21 +2,16 @@ module Shoryuken
   module Middleware
     module Server
       class AutoDelete
-        def call(worker, queue, sqs_msg, body)
+        def call(worker, queue, sqs_msg, _body)
           yield
 
-          auto_delete = worker.class.get_shoryuken_options['delete'] || worker.class.get_shoryuken_options['auto_delete']
+          return unless worker.class.auto_delete?
 
-          if auto_delete
-            entries = [sqs_msg].flatten.map.with_index do |message, i|
-              { id: i.to_s, receipt_handle: message.receipt_handle }
-            end
+          entries = [sqs_msg].flatten.map.with_index { |message, i| { id: i.to_s, receipt_handle: message.receipt_handle } }
 
-            Shoryuken::Client.queues(queue).delete_messages(entries: entries)
-          end
+          Shoryuken::Client.queues(queue).delete_messages(entries: entries)
         end
       end
     end
   end
 end
-

data/lib/shoryuken/middleware/server/auto_extend_visibility.rb

@@ -7,6 +7,8 @@ module Shoryuken
         EXTEND_UPFRONT_SECONDS = 5
 
         def call(worker, queue, sqs_msg, body)
+          return yield unless worker.class.auto_visibility_timeout?
+
           if sqs_msg.is_a?(Array)
             logger.warn { "Auto extend visibility isn't supported for batch workers" }
             return yield
@@ -23,22 +25,19 @@ module Shoryuken
         class MessageVisibilityExtender
           include Util
 
-          def auto_extend(worker, queue, sqs_msg, body)
+          def auto_extend(_worker, queue, sqs_msg, _body)
             queue_visibility_timeout = Shoryuken::Client.queues(queue).visibility_timeout
 
             Concurrent::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
               begin
                 logger.debug do
-                  "Extending message #{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id} " \
-                  "visibility timeout by #{queue_visibility_timeout}s."
+                  "Extending message #{queue}/#{sqs_msg.message_id} visibility timeout by #{queue_visibility_timeout}s"
                 end
 
                 sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
-              rescue => e
+              rescue => ex
                 logger.error do
-                  'Could not auto extend the message ' \
-                  "#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id} " \
-                  "visibility timeout. Error: #{e.message}"
+                  "Could not auto extend the message #{queue}/#{sqs_msg.message_id} visibility timeout. Error: #{ex.message}"
                 end
               end
             end
@@ -46,8 +45,6 @@ module Shoryuken
         end
 
         def auto_visibility_timer(worker, queue, sqs_msg, body)
-          return unless worker.class.auto_visibility_timeout?
-
           MessageVisibilityExtender.new.auto_extend(worker, queue, sqs_msg, body).tap(&:execute)
         end
       end

data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb

@@ -4,7 +4,9 @@ module Shoryuken
       class ExponentialBackoffRetry
         include Util
 
-        def call(worker, queue, sqs_msg, body)
+        def call(worker, _queue, sqs_msg, _body)
+          return yield unless worker.class.exponential_backoff?
+
           if sqs_msg.is_a?(Array)
             logger.warn { "Exponential backoff isn't supported for batch workers" }
             return yield
@@ -12,7 +14,7 @@ module Shoryuken
 
           started_at = Time.now
           yield
-        rescue
+        rescue => ex
           retry_intervals = worker.class.get_shoryuken_options['retry_intervals']
 
           if retry_intervals.nil? || !handle_failure(sqs_msg, started_at, retry_intervals)
@@ -20,6 +22,10 @@ module Shoryuken
             # This allows custom middleware (like exception notifiers) to be aware of the unhandled failure.
             raise
           end
+
+          logger.warn { "Message #{sqs_msg.message_id} will attempt retry due to error: #{ex.message}" }
+          # since we didn't raise, lets log the backtrace for debugging purposes.
+          logger.debug { ex.backtrace.join("\n") } unless ex.backtrace.nil?
         end
 
         private
@@ -47,7 +53,7 @@ module Shoryuken
 
           sqs_msg.change_visibility(visibility_timeout: next_visibility_timeout(interval.to_i, started_at))
 
-          logger.info { "Message #{sqs_msg.message_id} failed, will be retried in #{interval} seconds." }
+          logger.info { "Message #{sqs_msg.message_id} failed, will be retried in #{interval} seconds" }
 
           true
         end

data/lib/shoryuken/middleware/server/timing.rb

@@ -4,27 +4,23 @@ module Shoryuken
       class Timing
         include Util
 
-        def call(worker, queue, sqs_msg, body)
-          Shoryuken::Logging.with_context("#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id}") do
-            begin
-              started_at = Time.now
+        def call(_worker, queue, _sqs_msg, _body)
+          started_at = Time.now
 
-              logger.info { "started at #{started_at}" }
+          logger.info { "started at #{started_at}" }
 
-              yield
+          yield
 
-              total_time = elapsed(started_at)
+          total_time = elapsed(started_at)
 
-              if (total_time / 1000.0) > (timeout = Shoryuken::Client.queues(queue).visibility_timeout)
-                logger.warn { "exceeded the queue visibility timeout by #{total_time - (timeout * 1000)} ms" }
-              end
-
-              logger.info { "completed in: #{total_time} ms" }
-            rescue => e
-              logger.info { "failed in: #{elapsed(started_at)} ms" }
-              raise e
-            end
+          if (total_time / 1000.0) > (timeout = Shoryuken::Client.queues(queue).visibility_timeout)
+            logger.warn { "exceeded the queue visibility timeout by #{total_time - (timeout * 1000)} ms" }
           end
+
+          logger.info { "completed in: #{total_time} ms" }
+        rescue
+          logger.info { "failed in: #{elapsed(started_at)} ms" }
+          raise
         end
       end
     end