shoryuken 3.0.11 → 3.1.0

data/lib/shoryuken/default_worker_registry.rb

@@ -24,7 +24,7 @@ module Shoryuken
                        @workers[queue]
                      end
 
-      worker_class.new
+      worker_class.new if worker_class
     end
 
     def queues
@@ -34,7 +34,7 @@ module Shoryuken
     def register_worker(queue, clazz)
       if (worker_class = @workers[queue])
         if worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true
-          fail ArgumentError, "Could not register #{clazz} for '#{queue}', "\
+          fail ArgumentError, "Could not register #{clazz} for #{queue}, "\
             "because #{worker_class} is already registered for this queue, "\
             "and Shoryuken doesn't support a batchable worker for a queue with multiple workers"
         end

data/lib/shoryuken/environment_loader.rb

@@ -44,7 +44,11 @@ module Shoryuken
 
       fail ArgumentError, "The supplied config file #{path} does not exist" unless File.exist?(path)
 
-      YAML.load(ERB.new(IO.read(path)).result).deep_symbolize_keys
+      if result = YAML.load(ERB.new(IO.read(path)).result)
+        result.deep_symbolize_keys
+      else
+        {}
+      end
     end
 
     def initialize_logger
@@ -96,13 +100,24 @@ module Shoryuken
       end
     end
 
-    def parse_queue(queue, weight = nil)
-      Shoryuken.add_queue(queue, [weight.to_i, 1].max)
+    def parse_queue(queue, weight = nil, group)
+      Shoryuken.add_queue(queue, [weight.to_i, 1].max, group)
     end
 
     def parse_queues
-      Shoryuken.options[:queues].to_a.each do |queue_and_weight|
-        parse_queue(*queue_and_weight)
+      if Shoryuken.options[:queues].to_a.any?
+        Shoryuken.add_group('default', Shoryuken.options.fetch(:concurrency, 25))
+
+        Shoryuken.options[:queues].to_a.each do |queue, weight|
+          parse_queue(queue, weight, 'default')
+        end
+      end
+
+      Shoryuken.options[:groups].to_a.each do |group, options|
+        Shoryuken.add_group(group, options.fetch(:concurrency, 25))
+        options[:queues].to_a.each do |queue, weight|
+          parse_queue(queue, weight, group)
+        end
       end
     end
 
@@ -119,10 +134,11 @@ module Shoryuken
     end
 
     def validate_queues
-      return Shoryuken.logger.warn { 'No queues supplied' } if Shoryuken.queues.empty?
+      return Shoryuken.logger.warn { 'No queues supplied' } if Shoryuken.ungrouped_queues.empty?
+
       non_existent_queues = []
 
-      Shoryuken.queues.uniq.each do |queue|
+      Shoryuken.ungrouped_queues.uniq.each do |queue|
         begin
           Shoryuken::Client.queues(queue)
         rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue
@@ -134,14 +150,14 @@ module Shoryuken
 
       fail(
         ArgumentError,
-        "The specified queue(s) #{non_existent_queues.join(', ')} do not exist.\nCheck 'shoryuken sqs create QUEUE-NAME' for creating a queue with default settings"
+        "The specified queue(s) #{non_existent_queues.join(', ')} do not exist.\nTry 'shoryuken sqs create QUEUE-NAME' for creating a queue with default settings"
       )
     end
 
     def validate_workers
       return if defined?(::ActiveJob)
 
-      all_queues = Shoryuken.queues
+      all_queues = Shoryuken.ungrouped_queues
       queues_with_workers = Shoryuken.worker_registry.queues
 
       (all_queues - queues_with_workers).each do |queue|

data/lib/shoryuken/fetcher.rb

@@ -4,23 +4,23 @@ module Shoryuken
 
     FETCH_LIMIT = 10
 
-    def fetch(queue, available_processors)
+    attr_reader :group
+
+    def initialize(group)
+      @group = group
+    end
+
+    def fetch(queue, limit)
       started_at = Time.now
 
       logger.debug { "Looking for new messages in #{queue}" }
 
-      begin
-        limit = available_processors > FETCH_LIMIT ? FETCH_LIMIT : available_processors
-
-        sqs_msgs = Array(receive_messages(queue, limit))
-        logger.info { "Found #{sqs_msgs.size} messages for #{queue.name}" } unless sqs_msgs.empty?
-        logger.debug { "Fetcher for #{queue} completed in #{elapsed(started_at)} ms" }
-        sqs_msgs
-      rescue => ex
-        logger.error { "Error fetching message: #{ex.message}" }
-        logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
-        []
-      end
+      sqs_msgs = Array(receive_messages(queue, [FETCH_LIMIT, limit].min))
+
+      logger.info { "Found #{sqs_msgs.size} messages for #{queue.name}" } unless sqs_msgs.empty?
+      logger.debug { "Fetcher for #{queue} completed in #{elapsed(started_at)} ms" }
+
+      sqs_msgs
     end
 
     private
@@ -29,10 +29,11 @@ module Shoryuken
       # AWS limits the batch size by 10
       limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
 
-      options = Shoryuken.sqs_client_receive_message_opts.to_h.dup
-      options[:max_number_of_messages] = limit
+      options = Shoryuken.sqs_client_receive_message_opts[group].to_h.dup
+
+      options[:max_number_of_messages]  = limit
       options[:message_attribute_names] = %w(All)
-      options[:attribute_names] = %w(All)
+      options[:attribute_names]         = %w(All)
 
       options.merge!(queue.options)
 

data/lib/shoryuken/launcher.rb

@@ -3,17 +3,96 @@ module Shoryuken
     include Util
 
     def initialize
-      @manager = Shoryuken::Manager.new(Shoryuken::Fetcher.new,
-                                        Shoryuken.options[:polling_strategy].new(Shoryuken.queues))
+      @managers    = create_managers
+      @shutdowning = Concurrent::AtomicBoolean.new(false)
     end
 
-    def stop(options = {})
-      @manager.stop(shutdown: !options[:shutdown].nil?,
-                    timeout: Shoryuken.options[:timeout])
+    def start
+      logger.info { 'Starting' }
+
+      start_callback
+      start_managers
+    end
+
+    def stop!
+      initiate_stop
+
+      executor.shutdown
+
+      return if executor.wait_for_termination(Shoryuken.options[:timeout])
+
+      executor.kill
+    end
+
+    def stop
+      fire_event(:quiet, true)
+
+      initiate_stop
+
+      executor.shutdown
+      executor.wait_for_termination
+    end
+
+    private
+
+    def executor
+      Concurrent.global_io_executor
+    end
+
+    def start_managers
+      @managers.each do |manager|
+        Concurrent::Promise.execute { manager.start }.rescue do |ex|
+          log_manager_failure(ex)
+          start_soft_shutdown
+        end
+      end
+    end
+
+    def start_soft_shutdown
+      Process.kill('USR1', Process.pid) if @shutdowning.make_true
+    end
+
+    def log_manager_failure(ex)
+      return unless ex
+
+      logger.error { "Manager failed: #{ex.message}" }
+      logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
+    end
+
+    def initiate_stop
+      logger.info { 'Shutting down' }
+
+      @managers.each(&:stop)
+
+      stop_callback
+    end
+
+    def start_callback
+      if (callback = Shoryuken.start_callback)
+        logger.debug { 'Calling start_callback' }
+        callback.call
+      end
+
+      fire_event(:startup)
+    end
+
+    def stop_callback
+      if (callback = Shoryuken.stop_callback)
+        logger.debug { 'Calling stop_callback' }
+        callback.call
+      end
+
+      fire_event(:shutdown, true)
     end
 
-    def run
-      @manager.start
+    def create_managers
+      Shoryuken.groups.map do |group, options|
+        Shoryuken::Manager.new(
+          Shoryuken::Fetcher.new(group),
+          Shoryuken.polling_strategy(group).new(options[:queues]),
+          options[:concurrency]
+        )
+      end
     end
   end
 end

data/lib/shoryuken/manager.rb

@@ -6,95 +6,71 @@ module Shoryuken
     # See https://github.com/phstc/shoryuken/issues/348#issuecomment-292847028
     MIN_DISPATCH_INTERVAL = 0.1
 
-    def initialize(fetcher, polling_strategy)
-      @count = Shoryuken.options.fetch(:concurrency, 25)
-
-      raise(ArgumentError, "Concurrency value #{@count} is invalid, it needs to be a positive number") unless @count > 0
-
-      @queues = Shoryuken.queues.dup.uniq
-
-      @done = Concurrent::AtomicBoolean.new(false)
-
-      @fetcher = fetcher
+    def initialize(fetcher, polling_strategy, concurrency)
+      @fetcher          = fetcher
       @polling_strategy = polling_strategy
-
-      @pool = Concurrent::FixedThreadPool.new(@count, max_queue: @count)
-      @dispatcher_executor = Concurrent::SingleThreadExecutor.new
+      @max_processors   = concurrency
+      @busy_processors  = Concurrent::AtomicFixnum.new(0)
+      @done             = Concurrent::AtomicBoolean.new(false)
     end
 
     def start
-      logger.info { 'Starting' }
-
-      dispatch_async
+      dispatch
     end
 
-    def stop(options = {})
+    def stop
       @done.make_true
-
-      if (callback = Shoryuken.stop_callback)
-        logger.info { 'Calling on_stop callback' }
-        callback.call
-      end
-
-      fire_event(:shutdown, true)
-
-      logger.info { 'Shutting down workers' }
-
-      @dispatcher_executor.kill
-
-      if options[:shutdown]
-        hard_shutdown_in(options[:timeout])
-      else
-        soft_shutdown
-      end
     end
 
-    def processor_failed(ex)
-      logger.error { "Processor failed: #{ex.message}" }
-      logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
-    end
+    private
 
-    def processor_done(queue)
-      logger.debug { "Process done for #{queue}" }
+    def stopped?
+      @done.true? || !Concurrent.global_io_executor.running?
     end
 
-    private
+    def dispatch
+      return if stopped?
 
-    def dispatch_async
-      @dispatcher_executor.post(&method(:dispatch_now))
-    end
+      if !ready.positive? || (queue = @polling_strategy.next_queue).nil?
+        return dispatch_later
+      end
 
-    def dispatch_now
-      return if @done.true?
+      fire_event(:dispatch)
 
-      begin
-        if ready.zero? || (queue = @polling_strategy.next_queue).nil?
-          sleep MIN_DISPATCH_INTERVAL
-          return
-        end
+      logger.info { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
 
-        fire_event(:dispatch)
+      batched_queue?(queue) ? dispatch_batch(queue) : dispatch_single_messages(queue)
 
-        logger.debug { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
+      dispatch
+    end
 
-        batched_queue?(queue) ? dispatch_batch(queue) : dispatch_single_messages(queue)
-      ensure
-        dispatch_async
-      end
+    def dispatch_later
+      sleep(MIN_DISPATCH_INTERVAL)
+      dispatch
     end
 
     def busy
-      @count - ready
+      @busy_processors.value
     end
 
     def ready
-      @pool.remaining_capacity
+      @max_processors - busy
     end
 
-    def assign(queue, sqs_msg)
+    def processor_done
+      @busy_processors.decrement
+    end
+
+    def assign(queue_name, sqs_msg)
+      return if stopped?
+
       logger.debug { "Assigning #{sqs_msg.message_id}" }
 
-      @pool.post { Processor.new(self).process(queue, sqs_msg) }
+      @busy_processors.increment
+
+      Concurrent::Promise.execute {
+        Processor.new(queue_name, sqs_msg).process
+      }.then { processor_done }.rescue { processor_done }
     end
 
     def dispatch_batch(queue)
@@ -105,6 +81,7 @@ module Shoryuken
 
     def dispatch_single_messages(queue)
       messages = @fetcher.fetch(queue, ready)
+
       @polling_strategy.messages_found(queue.name, messages.size)
       messages.each { |message| assign(queue.name, message) }
     end
@@ -113,24 +90,6 @@ module Shoryuken
       Shoryuken.worker_registry.batch_receive_messages?(queue.name)
     end
 
-    def soft_shutdown
-      @pool.shutdown
-      @pool.wait_for_termination
-    end
-
-    def hard_shutdown_in(delay)
-      if busy > 0
-        logger.info { "Pausing up to #{delay} seconds to allow workers to finish..." }
-      end
-
-      @pool.shutdown
-
-      return if @pool.wait_for_termination(delay)
-
-      logger.info { "Hard shutting down #{busy} busy workers" }
-      @pool.kill
-    end
-
     def patch_batch!(sqs_msgs)
       sqs_msgs.instance_eval do
         def message_id

data/lib/shoryuken/options.rb

@@ -0,0 +1,192 @@
+module Shoryuken
+  class Options
+    DEFAULTS = {
+      concurrency: 25,
+      queues: [],
+      aws: {},
+      delay: 0,
+      timeout: 8,
+      lifecycle_events: {
+        startup: [],
+        dispatch: [],
+        quiet: [],
+        shutdown: []
+      }
+    }.freeze
+
+    @@groups                          = {}
+    @@worker_registry                 = DefaultWorkerRegistry.new
+    @@active_job_queue_name_prefixing = false
+    @@sqs_client                      = nil
+    @@sqs_client_receive_message_opts = {}
+    @@start_callback                  = nil
+    @@stop_callback                   = nil
+
+    class << self
+      def add_group(group, concurrency)
+        groups[group] ||= {
+          concurrency: concurrency,
+          queues: []
+        }
+      end
+
+      def groups
+        @@groups
+      end
+
+      def add_queue(queue, weight, group)
+        weight.times do
+          groups[group][:queues] << queue
+        end
+      end
+
+      def ungrouped_queues
+        groups.values.flat_map { |options| options[:queues] }
+      end
+
+      def worker_registry
+        @@worker_registry
+      end
+
+      def worker_registry=(worker_registry)
+        @@worker_registry = worker_registry
+      end
+
+      def polling_strategy(group)
+        options[group].to_h.fetch(:polling_strategy, Polling::WeightedRoundRobin)
+      end
+
+      def start_callback
+        @@start_callback
+      end
+
+      def start_callback=(start_callback)
+        @@start_callback = start_callback
+      end
+
+      def stop_callback
+        @@stop_callback
+      end
+
+      def stop_callback=(stop_callback)
+        @@stop_callback = stop_callback
+      end
+
+      def active_job_queue_name_prefixing
+        @@active_job_queue_name_prefixing
+      end
+
+      def active_job_queue_name_prefixing=(active_job_queue_name_prefixing)
+        @@active_job_queue_name_prefixing = active_job_queue_name_prefixing
+      end
+
+      def sqs_client
+        @@sqs_client ||= Aws::SQS::Client.new
+      end
+
+      def sqs_client=(sqs_client)
+        @@sqs_client = sqs_client
+      end
+
+      def sqs_client_receive_message_opts
+        @@sqs_client_receive_message_opts
+      end
+
+      def sqs_client_receive_message_opts=(sqs_client_receive_message_opts)
+        @@sqs_client_receive_message_opts['default'] = sqs_client_receive_message_opts
+      end
+
+      def options
+        @@options ||= DEFAULTS.dup
+      end
+
+      def logger
+        Shoryuken::Logging.logger
+      end
+
+      def register_worker(*args)
+        @@worker_registry.register_worker(*args)
+      end
+
+      def configure_server
+        yield self if server?
+      end
+
+      def server_middleware
+        @@server_chain ||= default_server_middleware
+        yield @@server_chain if block_given?
+        @@server_chain
+      end
+
+      def configure_client
+        yield self unless server?
+      end
+
+      def client_middleware
+        @@client_chain ||= default_client_middleware
+        yield @@client_chain if block_given?
+        @@client_chain
+      end
+
+      def default_worker_options
+        @@default_worker_options ||= {
+          'queue'                   => 'default',
+          'delete'                  => false,
+          'auto_delete'             => false,
+          'auto_visibility_timeout' => false,
+          'retry_intervals'         => nil,
+          'batch'                   => false
+        }
+      end
+
+      def default_worker_options=(default_worker_options)
+        @@default_worker_options = default_worker_options
+      end
+
+      def on_start(&block)
+        @@start_callback = block
+      end
+
+      def on_stop(&block)
+        @@stop_callback = block
+      end
+
+      # Register a block to run at a point in the Shoryuken lifecycle.
+      # :startup, :quiet or :shutdown are valid events.
+      #
+      #   Shoryuken.configure_server do |config|
+      #     config.on(:shutdown) do
+      #       puts "Goodbye cruel world!"
+      #     end
+      #   end
+      def on(event, &block)
+        fail ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
+        fail ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
+        options[:lifecycle_events][event] << block
+      end
+
+      private
+
+      def default_server_middleware
+        Middleware::Chain.new do |m|
+          m.add Middleware::Server::Timing
+          m.add Middleware::Server::ExponentialBackoffRetry
+          m.add Middleware::Server::AutoDelete
+          m.add Middleware::Server::AutoExtendVisibility
+          if defined?(::ActiveRecord::Base)
+            require 'shoryuken/middleware/server/active_record'
+            m.add Middleware::Server::ActiveRecord
+          end
+        end
+      end
+
+      def default_client_middleware
+        Middleware::Chain.new
+      end
+
+      def server?
+        defined?(Shoryuken::CLI)
+      end
+    end
+  end
+end