shoryuken 3.0.4 → 3.1.0

data/lib/shoryuken/manager.rb

@@ -3,97 +3,85 @@ module Shoryuken
     include Util
 
     BATCH_LIMIT = 10
+    # See https://github.com/phstc/shoryuken/issues/348#issuecomment-292847028
+    MIN_DISPATCH_INTERVAL = 0.1
 
-    def initialize(fetcher, polling_strategy)
-      @count = Shoryuken.options.fetch(:concurrency, 25)
-
-      raise(ArgumentError, "Concurrency value #{@count} is invalid, it needs to be a positive number") unless @count > 0
-
-      @queues = Shoryuken.queues.dup.uniq
-
-      @done = Concurrent::AtomicBoolean.new(false)
-
-      @fetcher = fetcher
+    def initialize(fetcher, polling_strategy, concurrency)
+      @fetcher          = fetcher
       @polling_strategy = polling_strategy
-
-      @pool = Concurrent::FixedThreadPool.new(@count, max_queue: @count)
-      @dispatcher_executor = Concurrent::SingleThreadExecutor.new
+      @max_processors   = concurrency
+      @busy_processors  = Concurrent::AtomicFixnum.new(0)
+      @done             = Concurrent::AtomicBoolean.new(false)
     end
 
     def start
-      logger.info { 'Starting' }
-
-      dispatch_async
+      dispatch
     end
 
-    def stop(options = {})
+    def stop
       @done.make_true
+    end
 
-      if (callback = Shoryuken.stop_callback)
-        logger.info { 'Calling Shoryuken.on_stop block' }
-        callback.call
-      end
-
-      fire_event(:shutdown, true)
-
-      logger.info { 'Shutting down workers' }
-
-      @dispatcher_executor.kill
+    private
 
-      if options[:shutdown]
-        hard_shutdown_in(options[:timeout])
-      else
-        soft_shutdown
-      end
+    def stopped?
+      @done.true? || !Concurrent.global_io_executor.running?
     end
 
-    def processor_done(queue)
-      logger.debug { "Process done for '#{queue}'" }
-    end
+    def dispatch
+      return if stopped?
 
-    private
+      if !ready.positive? || (queue = @polling_strategy.next_queue).nil?
+        return dispatch_later
+      end
 
-    def dispatch_async
-      @dispatcher_executor.post(&method(:dispatch_now))
-    end
+      fire_event(:dispatch)
 
-    def dispatch_now
-      return if @done.true?
+      logger.info { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
 
-      begin
-        return if ready.zero?
-        return unless (queue = @polling_strategy.next_queue)
+      batched_queue?(queue) ? dispatch_batch(queue) : dispatch_single_messages(queue)
 
-        logger.debug { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
+      dispatch
+    end
 
-        batched_queue?(queue) ? dispatch_batch(queue) : dispatch_single_messages(queue)
-      ensure
-        dispatch_async
-      end
+    def dispatch_later
+      sleep(MIN_DISPATCH_INTERVAL)
+      dispatch
     end
 
     def busy
-      @count - ready
+      @busy_processors.value
     end
 
     def ready
-      @pool.remaining_capacity
+      @max_processors - busy
+    end
+
+    def processor_done
+      @busy_processors.decrement
     end
 
-    def assign(queue, sqs_msg)
+    def assign(queue_name, sqs_msg)
+      return if stopped?
+
       logger.debug { "Assigning #{sqs_msg.message_id}" }
 
-      @pool.post { Processor.new(self).process(queue, sqs_msg) }
+      @busy_processors.increment
+
+      Concurrent::Promise.execute {
+        Processor.new(queue_name, sqs_msg).process
+      }.then { processor_done }.rescue { processor_done }
     end
 
     def dispatch_batch(queue)
-      batch = @fetcher.fetch(queue, BATCH_LIMIT)
+      return if (batch = @fetcher.fetch(queue, BATCH_LIMIT)).none?
       @polling_strategy.messages_found(queue.name, batch.size)
       assign(queue.name, patch_batch!(batch))
     end
 
     def dispatch_single_messages(queue)
       messages = @fetcher.fetch(queue, ready)
+
       @polling_strategy.messages_found(queue.name, messages.size)
       messages.each { |message| assign(queue.name, message) }
     end
@@ -102,24 +90,6 @@ module Shoryuken
       Shoryuken.worker_registry.batch_receive_messages?(queue.name)
     end
 
-    def soft_shutdown
-      @pool.shutdown
-      @pool.wait_for_termination
-    end
-
-    def hard_shutdown_in(delay)
-      if busy > 0
-        logger.info { "Pausing up to #{delay} seconds to allow workers to finish..." }
-      end
-
-      @pool.shutdown
-
-      return if @pool.wait_for_termination(delay)
-
-      logger.info { "Hard shutting down #{busy} busy workers" }
-      @pool.kill
-    end
-
     def patch_batch!(sqs_msgs)
       sqs_msgs.instance_eval do
         def message_id

data/lib/shoryuken/middleware/server/auto_delete.rb

@@ -5,18 +5,13 @@ module Shoryuken
         def call(worker, queue, sqs_msg, body)
           yield
 
-          auto_delete = worker.class.get_shoryuken_options['delete'] || worker.class.get_shoryuken_options['auto_delete']
+          return unless worker.class.auto_delete?
 
-          if auto_delete
-            entries = [sqs_msg].flatten.map.with_index do |message, i|
-              { id: i.to_s, receipt_handle: message.receipt_handle }
-            end
+          entries = [sqs_msg].flatten.map.with_index { |message, i| { id: i.to_s, receipt_handle: message.receipt_handle } }
 
-            Shoryuken::Client.queues(queue).delete_messages(entries: entries)
-          end
+          Shoryuken::Client.queues(queue).delete_messages(entries: entries)
         end
       end
     end
   end
 end
-

data/lib/shoryuken/middleware/server/auto_extend_visibility.rb

@@ -7,6 +7,8 @@ module Shoryuken
         EXTEND_UPFRONT_SECONDS = 5
 
         def call(worker, queue, sqs_msg, body)
+          return yield unless worker.class.auto_visibility_timeout?
+
           if sqs_msg.is_a?(Array)
             logger.warn { "Auto extend visibility isn't supported for batch workers" }
             return yield
@@ -34,11 +36,11 @@ module Shoryuken
                 end
 
                 sqs_msg.change_visibility(visibility_timeout: queue_visibility_timeout)
-              rescue => e
+              rescue => ex
                 logger.error do
                   'Could not auto extend the message ' \
                   "#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id} " \
-                  "visibility timeout. Error: #{e.message}"
+                  "visibility timeout. Error: #{ex.message}"
                 end
               end
             end
@@ -46,8 +48,6 @@ module Shoryuken
         end
 
         def auto_visibility_timer(worker, queue, sqs_msg, body)
-          return unless worker.class.auto_visibility_timeout?
-
           MessageVisibilityExtender.new.auto_extend(worker, queue, sqs_msg, body).tap(&:execute)
         end
       end

data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb

@@ -5,6 +5,8 @@ module Shoryuken
         include Util
 
         def call(worker, queue, sqs_msg, body)
+          return yield unless worker.class.exponential_backoff?
+
           if sqs_msg.is_a?(Array)
             logger.warn { "Exponential backoff isn't supported for batch workers" }
             return yield
@@ -12,7 +14,7 @@ module Shoryuken
 
           started_at = Time.now
           yield
-        rescue
+        rescue => ex
           retry_intervals = worker.class.get_shoryuken_options['retry_intervals']
 
           if retry_intervals.nil? || !handle_failure(sqs_msg, started_at, retry_intervals)
@@ -20,6 +22,10 @@ module Shoryuken
             # This allows custom middleware (like exception notifiers) to be aware of the unhandled failure.
             raise
           end
+
+          logger.warn { "Message #{sqs_msg.message_id} will attempt retry due to error: #{ex.message}" }
+          # since we didn't raise, lets log the backtrace for debugging purposes.
+          logger.debug { ex.backtrace.join("\n") } unless ex.backtrace.nil?
         end
 
         private
@@ -47,7 +53,7 @@ module Shoryuken
 
           sqs_msg.change_visibility(visibility_timeout: next_visibility_timeout(interval.to_i, started_at))
 
-          logger.info { "Message #{sqs_msg.message_id} failed, will be retried in #{interval} seconds." }
+          logger.info { "Message #{sqs_msg.message_id} failed, will be retried in #{interval} seconds" }
 
           true
         end

data/lib/shoryuken/middleware/server/timing.rb

@@ -20,9 +20,9 @@ module Shoryuken
               end
 
               logger.info { "completed in: #{total_time} ms" }
-            rescue => e
+            rescue
               logger.info { "failed in: #{elapsed(started_at)} ms" }
-              raise e
+              raise
             end
           end
         end

data/lib/shoryuken/options.rb

@@ -0,0 +1,192 @@
+module Shoryuken
+  class Options
+    DEFAULTS = {
+      concurrency: 25,
+      queues: [],
+      aws: {},
+      delay: 0,
+      timeout: 8,
+      lifecycle_events: {
+        startup: [],
+        dispatch: [],
+        quiet: [],
+        shutdown: []
+      }
+    }.freeze
+
+    @@groups                          = {}
+    @@worker_registry                 = DefaultWorkerRegistry.new
+    @@active_job_queue_name_prefixing = false
+    @@sqs_client                      = nil
+    @@sqs_client_receive_message_opts = {}
+    @@start_callback                  = nil
+    @@stop_callback                   = nil
+
+    class << self
+      def add_group(group, concurrency)
+        groups[group] ||= {
+          concurrency: concurrency,
+          queues: []
+        }
+      end
+
+      def groups
+        @@groups
+      end
+
+      def add_queue(queue, weight, group)
+        weight.times do
+          groups[group][:queues] << queue
+        end
+      end
+
+      def ungrouped_queues
+        groups.values.flat_map { |options| options[:queues] }
+      end
+
+      def worker_registry
+        @@worker_registry
+      end
+
+      def worker_registry=(worker_registry)
+        @@worker_registry = worker_registry
+      end
+
+      def polling_strategy(group)
+        options[group].to_h.fetch(:polling_strategy, Polling::WeightedRoundRobin)
+      end
+
+      def start_callback
+        @@start_callback
+      end
+
+      def start_callback=(start_callback)
+        @@start_callback = start_callback
+      end
+
+      def stop_callback
+        @@stop_callback
+      end
+
+      def stop_callback=(stop_callback)
+        @@stop_callback = stop_callback
+      end
+
+      def active_job_queue_name_prefixing
+        @@active_job_queue_name_prefixing
+      end
+
+      def active_job_queue_name_prefixing=(active_job_queue_name_prefixing)
+        @@active_job_queue_name_prefixing = active_job_queue_name_prefixing
+      end
+
+      def sqs_client
+        @@sqs_client ||= Aws::SQS::Client.new
+      end
+
+      def sqs_client=(sqs_client)
+        @@sqs_client = sqs_client
+      end
+
+      def sqs_client_receive_message_opts
+        @@sqs_client_receive_message_opts
+      end
+
+      def sqs_client_receive_message_opts=(sqs_client_receive_message_opts)
+        @@sqs_client_receive_message_opts['default'] = sqs_client_receive_message_opts
+      end
+
+      def options
+        @@options ||= DEFAULTS.dup
+      end
+
+      def logger
+        Shoryuken::Logging.logger
+      end
+
+      def register_worker(*args)
+        @@worker_registry.register_worker(*args)
+      end
+
+      def configure_server
+        yield self if server?
+      end
+
+      def server_middleware
+        @@server_chain ||= default_server_middleware
+        yield @@server_chain if block_given?
+        @@server_chain
+      end
+
+      def configure_client
+        yield self unless server?
+      end
+
+      def client_middleware
+        @@client_chain ||= default_client_middleware
+        yield @@client_chain if block_given?
+        @@client_chain
+      end
+
+      def default_worker_options
+        @@default_worker_options ||= {
+          'queue'                   => 'default',
+          'delete'                  => false,
+          'auto_delete'             => false,
+          'auto_visibility_timeout' => false,
+          'retry_intervals'         => nil,
+          'batch'                   => false
+        }
+      end
+
+      def default_worker_options=(default_worker_options)
+        @@default_worker_options = default_worker_options
+      end
+
+      def on_start(&block)
+        @@start_callback = block
+      end
+
+      def on_stop(&block)
+        @@stop_callback = block
+      end
+
+      # Register a block to run at a point in the Shoryuken lifecycle.
+      # :startup, :quiet or :shutdown are valid events.
+      #
+      #   Shoryuken.configure_server do |config|
+      #     config.on(:shutdown) do
+      #       puts "Goodbye cruel world!"
+      #     end
+      #   end
+      def on(event, &block)
+        fail ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
+        fail ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
+        options[:lifecycle_events][event] << block
+      end
+
+      private
+
+      def default_server_middleware
+        Middleware::Chain.new do |m|
+          m.add Middleware::Server::Timing
+          m.add Middleware::Server::ExponentialBackoffRetry
+          m.add Middleware::Server::AutoDelete
+          m.add Middleware::Server::AutoExtendVisibility
+          if defined?(::ActiveRecord::Base)
+            require 'shoryuken/middleware/server/active_record'
+            m.add Middleware::Server::ActiveRecord
+          end
+        end
+      end
+
+      def default_client_middleware
+        Middleware::Chain.new
+      end
+
+      def server?
+        defined?(Shoryuken::CLI)
+      end
+    end
+  end
+end

data/lib/shoryuken/polling/base.rb

@@ -0,0 +1,67 @@
+module Shoryuken
+  module Polling
+    QueueConfiguration = Struct.new(:name, :options) do
+      def hash
+        name.hash
+      end
+
+      def ==(other)
+        case other
+        when String
+          if options.empty?
+            name == other
+          else
+            false
+          end
+        else
+          super
+        end
+      end
+
+      alias_method :eql?, :==
+
+      def to_s
+        if options.empty?
+          name
+        else
+          "#<QueueConfiguration #{name} options=#{options.inspect}>"
+        end
+      end
+    end
+
+    class BaseStrategy
+      include Util
+
+      def next_queue
+        fail NotImplementedError
+      end
+
+      def messages_found(queue, messages_found)
+        fail NotImplementedError
+      end
+
+      def active_queues
+        fail NotImplementedError
+      end
+
+      def ==(other)
+        case other
+        when Array
+          @queues == other
+        else
+          if other.respond_to?(:active_queues)
+            active_queues == other.active_queues
+          else
+            false
+          end
+        end
+      end
+
+      private
+
+      def delay
+        Shoryuken.options[:delay].to_f
+      end
+    end
+  end
+end

data/lib/shoryuken/polling/strict_priority.rb

@@ -0,0 +1,77 @@
+module Shoryuken
+  module Polling
+    class StrictPriority < BaseStrategy
+      def initialize(queues)
+        # Priority ordering of the queues, highest priority first
+        @queues = queues
+        .group_by { |q| q }
+        .sort_by { |_, qs| -qs.count }
+        .map(&:first)
+
+        # Pause status of the queues, default to past time (unpaused)
+        @paused_until = queues
+        .each_with_object(Hash.new) { |queue, h| h[queue] = Time.at(0) }
+
+        # Start queues at 0
+        reset_next_queue
+      end
+
+      def next_queue
+        next_queue = next_active_queue
+        next_queue.nil? ? nil : QueueConfiguration.new(next_queue, {})
+      end
+
+      def messages_found(queue, messages_found)
+        if messages_found == 0
+          pause(queue)
+        else
+          reset_next_queue
+        end
+      end
+
+      def active_queues
+        @queues
+        .reverse
+        .map.with_index(1)
+        .reject { |q, _| queue_paused?(q) }
+        .reverse
+      end
+
+      private
+
+      def next_active_queue
+        reset_next_queue if queues_unpaused_since?
+
+        size = @queues.length
+        size.times do
+          queue = @queues[@next_queue_index]
+          @next_queue_index = (@next_queue_index + 1) % size
+          return queue unless queue_paused?(queue)
+        end
+
+        nil
+      end
+
+      def queues_unpaused_since?
+        last = @last_unpause_check
+        now = @last_unpause_check = Time.now
+
+        last && @paused_until.values.any? { |t| t > last && t <= now }
+      end
+
+      def reset_next_queue
+        @next_queue_index = 0
+      end
+
+      def queue_paused?(queue)
+        @paused_until[queue] > Time.now
+      end
+
+      def pause(queue)
+        return unless delay > 0
+        @paused_until[queue] = Time.now + delay
+        logger.debug "Paused #{queue}"
+      end
+    end
+  end
+end

data/lib/shoryuken/polling/weighted_round_robin.rb

@@ -0,0 +1,66 @@
+module Shoryuken
+  module Polling
+    class WeightedRoundRobin < BaseStrategy
+      def initialize(queues)
+        @initial_queues = queues
+        @queues = queues.dup.uniq
+        @paused_queues = []
+      end
+
+      def next_queue
+        unpause_queues
+        queue = @queues.shift
+        return nil if queue.nil?
+
+        @queues << queue
+        QueueConfiguration.new(queue, {})
+      end
+
+      def messages_found(queue, messages_found)
+        if messages_found == 0
+          pause(queue)
+          return
+        end
+
+        maximum_weight = maximum_queue_weight(queue)
+        current_weight = current_queue_weight(queue)
+        if maximum_weight > current_weight
+          logger.info { "Increasing #{queue} weight to #{current_weight + 1}, max: #{maximum_weight}" }
+          @queues << queue
+        end
+      end
+
+      def active_queues
+        unparse_queues(@queues)
+      end
+
+      private
+
+      def pause(queue)
+        return unless @queues.delete(queue)
+        @paused_queues << [Time.now + delay, queue]
+        logger.debug "Paused #{queue}"
+      end
+
+      def unpause_queues
+        return if @paused_queues.empty?
+        return if Time.now < @paused_queues.first[0]
+        pause = @paused_queues.shift
+        @queues << pause[1]
+        logger.debug "Unpaused #{pause[1]}"
+      end
+
+      def current_queue_weight(queue)
+        queue_weight(@queues, queue)
+      end
+
+      def maximum_queue_weight(queue)
+        queue_weight(@initial_queues, queue)
+      end
+
+      def queue_weight(queues, queue)
+        queues.count { |q| q == queue }
+      end
+    end
+  end
+end