shoryuken 3.0.6 → 4.0.0

data/lib/shoryuken/options.rb

@@ -0,0 +1,225 @@
+module Shoryuken
+  class Options
+    DEFAULTS = {
+      concurrency: 25,
+      queues: [],
+      aws: {},
+      delay: 0,
+      timeout: 8,
+      lifecycle_events: {
+        startup: [],
+        dispatch: [],
+        quiet: [],
+        shutdown: []
+      }
+    }.freeze
+
+    @@groups                          = {}
+    @@worker_registry                 = DefaultWorkerRegistry.new
+    @@active_job_queue_name_prefixing = false
+    @@sqs_client                      = nil
+    @@sqs_client_receive_message_opts = {}
+    @@start_callback                  = nil
+    @@stop_callback                   = nil
+    @@worker_executor                 = Worker::DefaultExecutor
+    @@launcher_executor               = nil
+
+    class << self
+      def active_job?
+        defined?(::ActiveJob)
+      end
+
+      def add_group(group, concurrency)
+        groups[group] ||= {
+          concurrency: concurrency,
+          queues: []
+        }
+      end
+
+      def groups
+        @@groups
+      end
+
+      def add_queue(queue, weight, group)
+        weight.times do
+          groups[group][:queues] << queue
+        end
+      end
+
+      def ungrouped_queues
+        groups.values.flat_map { |options| options[:queues] }
+      end
+
+      def worker_registry
+        @@worker_registry
+      end
+
+      def worker_registry=(worker_registry)
+        @@worker_registry = worker_registry
+      end
+
+      def worker_executor
+        @@worker_executor
+      end
+
+      def worker_executor=(worker_executor)
+        @@worker_executor = worker_executor
+      end
+
+      def launcher_executor
+        @@launcher_executor
+      end
+
+      def launcher_executor=(launcher_executor)
+        @@launcher_executor = launcher_executor
+      end
+
+      def polling_strategy(group)
+        strategy = (group == 'default' ? options : options[:groups].to_h[group]).to_h[:polling_strategy]
+
+        case strategy
+        when 'WeightedRoundRobin', nil # Default case
+          Polling::WeightedRoundRobin
+        when 'StrictPriority'
+          Polling::StrictPriority
+        when Class
+          strategy
+        else
+          raise ArgumentError, "#{strategy} is not a valid polling_strategy"
+        end
+      end
+
+      def start_callback
+        @@start_callback
+      end
+
+      def start_callback=(start_callback)
+        @@start_callback = start_callback
+      end
+
+      def stop_callback
+        @@stop_callback
+      end
+
+      def stop_callback=(stop_callback)
+        @@stop_callback = stop_callback
+      end
+
+      def active_job_queue_name_prefixing
+        @@active_job_queue_name_prefixing
+      end
+
+      def active_job_queue_name_prefixing=(active_job_queue_name_prefixing)
+        @@active_job_queue_name_prefixing = active_job_queue_name_prefixing
+      end
+
+      def sqs_client
+        @@sqs_client ||= Aws::SQS::Client.new
+      end
+
+      def sqs_client=(sqs_client)
+        @@sqs_client = sqs_client
+      end
+
+      def sqs_client_receive_message_opts
+        @@sqs_client_receive_message_opts
+      end
+
+      def sqs_client_receive_message_opts=(sqs_client_receive_message_opts)
+        @@sqs_client_receive_message_opts['default'] = sqs_client_receive_message_opts
+      end
+
+      def options
+        @@options ||= DEFAULTS.dup
+      end
+
+      def logger
+        Shoryuken::Logging.logger
+      end
+
+      def register_worker(*args)
+        @@worker_registry.register_worker(*args)
+      end
+
+      def configure_server
+        yield self if server?
+      end
+
+      def server_middleware
+        @@server_chain ||= default_server_middleware
+        yield @@server_chain if block_given?
+        @@server_chain
+      end
+
+      def configure_client
+        yield self unless server?
+      end
+
+      def client_middleware
+        @@client_chain ||= default_client_middleware
+        yield @@client_chain if block_given?
+        @@client_chain
+      end
+
+      def default_worker_options
+        @@default_worker_options ||= {
+          'queue'                   => 'default',
+          'delete'                  => false,
+          'auto_delete'             => false,
+          'auto_visibility_timeout' => false,
+          'retry_intervals'         => nil,
+          'batch'                   => false
+        }
+      end
+
+      def default_worker_options=(default_worker_options)
+        @@default_worker_options = default_worker_options
+      end
+
+      def on_start(&block)
+        @@start_callback = block
+      end
+
+      def on_stop(&block)
+        @@stop_callback = block
+      end
+
+      # Register a block to run at a point in the Shoryuken lifecycle.
+      # :startup, :quiet or :shutdown are valid events.
+      #
+      #   Shoryuken.configure_server do |config|
+      #     config.on(:shutdown) do
+      #       puts "Goodbye cruel world!"
+      #     end
+      #   end
+      def on(event, &block)
+        fail ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
+        fail ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
+        options[:lifecycle_events][event] << block
+      end
+
+      private
+
+      def default_server_middleware
+        Middleware::Chain.new do |m|
+          m.add Middleware::Server::Timing
+          m.add Middleware::Server::ExponentialBackoffRetry
+          m.add Middleware::Server::AutoDelete
+          m.add Middleware::Server::AutoExtendVisibility
+          if defined?(::ActiveRecord::Base)
+            require 'shoryuken/middleware/server/active_record'
+            m.add Middleware::Server::ActiveRecord
+          end
+        end
+      end
+
+      def default_client_middleware
+        Middleware::Chain.new
+      end
+
+      def server?
+        defined?(Shoryuken::CLI)
+      end
+    end
+  end
+end

data/lib/shoryuken/polling/base.rb

@@ -0,0 +1,67 @@
+module Shoryuken
+  module Polling
+    QueueConfiguration = Struct.new(:name, :options) do
+      def hash
+        name.hash
+      end
+
+      def ==(other)
+        case other
+        when String
+          if options.empty?
+            name == other
+          else
+            false
+          end
+        else
+          super
+        end
+      end
+
+      alias_method :eql?, :==
+
+      def to_s
+        if options.empty?
+          name
+        else
+          "#<QueueConfiguration #{name} options=#{options.inspect}>"
+        end
+      end
+    end
+
+    class BaseStrategy
+      include Util
+
+      def next_queue
+        fail NotImplementedError
+      end
+
+      def messages_found(_queue, _messages_found)
+        fail NotImplementedError
+      end
+
+      def active_queues
+        fail NotImplementedError
+      end
+
+      def ==(other)
+        case other
+        when Array
+          @queues == other
+        else
+          if other.respond_to?(:active_queues)
+            active_queues == other.active_queues
+          else
+            false
+          end
+        end
+      end
+
+      private
+
+      def delay
+        Shoryuken.options[:delay].to_f
+      end
+    end
+  end
+end

data/lib/shoryuken/polling/strict_priority.rb

@@ -0,0 +1,77 @@
+module Shoryuken
+  module Polling
+    class StrictPriority < BaseStrategy
+      def initialize(queues)
+        # Priority ordering of the queues, highest priority first
+        @queues = queues
+                  .group_by { |q| q }
+                  .sort_by { |_, qs| -qs.count }
+                  .map(&:first)
+
+        # Pause status of the queues, default to past time (unpaused)
+        @paused_until = queues
+                        .each_with_object({}) { |queue, h| h[queue] = Time.at(0) }
+
+        # Start queues at 0
+        reset_next_queue
+      end
+
+      def next_queue
+        next_queue = next_active_queue
+        next_queue.nil? ? nil : QueueConfiguration.new(next_queue, {})
+      end
+
+      def messages_found(queue, messages_found)
+        if messages_found == 0
+          pause(queue)
+        else
+          reset_next_queue
+        end
+      end
+
+      def active_queues
+        @queues
+          .reverse
+          .map.with_index(1)
+          .reject { |q, _| queue_paused?(q) }
+          .reverse
+      end
+
+      private
+
+      def next_active_queue
+        reset_next_queue if queues_unpaused_since?
+
+        size = @queues.length
+        size.times do
+          queue = @queues[@next_queue_index]
+          @next_queue_index = (@next_queue_index + 1) % size
+          return queue unless queue_paused?(queue)
+        end
+
+        nil
+      end
+
+      def queues_unpaused_since?
+        last = @last_unpause_check
+        now = @last_unpause_check = Time.now
+
+        last && @paused_until.values.any? { |t| t > last && t <= now }
+      end
+
+      def reset_next_queue
+        @next_queue_index = 0
+      end
+
+      def queue_paused?(queue)
+        @paused_until[queue] > Time.now
+      end
+
+      def pause(queue)
+        return unless delay > 0
+        @paused_until[queue] = Time.now + delay
+        logger.debug "Paused #{queue}"
+      end
+    end
+  end
+end

data/lib/shoryuken/polling/weighted_round_robin.rb

@@ -0,0 +1,66 @@
+module Shoryuken
+  module Polling
+    class WeightedRoundRobin < BaseStrategy
+      def initialize(queues)
+        @initial_queues = queues
+        @queues = queues.dup.uniq
+        @paused_queues = []
+      end
+
+      def next_queue
+        unpause_queues
+        queue = @queues.shift
+        return nil if queue.nil?
+
+        @queues << queue
+        QueueConfiguration.new(queue, {})
+      end
+
+      def messages_found(queue, messages_found)
+        if messages_found == 0
+          pause(queue)
+          return
+        end
+
+        maximum_weight = maximum_queue_weight(queue)
+        current_weight = current_queue_weight(queue)
+        if maximum_weight > current_weight
+          logger.info { "Increasing #{queue} weight to #{current_weight + 1}, max: #{maximum_weight}" }
+          @queues << queue
+        end
+      end
+
+      def active_queues
+        unparse_queues(@queues)
+      end
+
+      private
+
+      def pause(queue)
+        return unless @queues.delete(queue)
+        @paused_queues << [Time.now + delay, queue]
+        logger.debug "Paused #{queue}"
+      end
+
+      def unpause_queues
+        return if @paused_queues.empty?
+        return if Time.now < @paused_queues.first[0]
+        pause = @paused_queues.shift
+        @queues << pause[1]
+        logger.debug "Unpaused #{pause[1]}"
+      end
+
+      def current_queue_weight(queue)
+        queue_weight(@queues, queue)
+      end
+
+      def maximum_queue_weight(queue)
+        queue_weight(@initial_queues, queue)
+      end
+
+      def queue_weight(queues, queue)
+        queues.count { |q| q == queue }
+      end
+    end
+  end
+end

data/lib/shoryuken/processor.rb

@@ -2,57 +2,48 @@ module Shoryuken
   class Processor
     include Util
 
-    def initialize(manager)
-      @manager = manager
+    attr_reader :queue, :sqs_msg
+
+    def self.process(queue, sqs_msg)
+      new(queue, sqs_msg).process
+    end
+
+    def initialize(queue, sqs_msg)
+      @queue   = queue
+      @sqs_msg = sqs_msg
     end
 
-    def process(queue, sqs_msg)
-      worker = Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
-      body = get_body(worker.class, sqs_msg)
+    def process
+      return logger.error { "No worker found for #{queue}" } unless worker
 
-      worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
-        worker.perform(sqs_msg, body)
+      Shoryuken::Logging.with_context("#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id}") do
+        worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
+          worker.perform(sqs_msg, body)
+        end
       end
     rescue Exception => ex
-      @manager.processor_failed(ex)
+      logger.error { "Processor failed: #{ex.message}" }
+      logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
+
       raise
-    ensure
-      @manager.processor_done(queue)
     end
 
     private
 
-    def get_body(worker_class, sqs_msg)
-      if sqs_msg.is_a? Array
-        sqs_msg.map { |m| parse_body(worker_class, m) }
-      else
-        parse_body(worker_class, sqs_msg)
-      end
+    def worker
+      @_worker ||= Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
     end
 
-    def parse_body(worker_class, sqs_msg)
-      body_parser = worker_class.get_shoryuken_options['body_parser']
-
-      case body_parser
-      when :json
-        JSON.parse(sqs_msg.body)
-      when Proc
-        body_parser.call(sqs_msg)
-      when :text, nil
-        sqs_msg.body
-      else
-        if body_parser.respond_to?(:parse)
-          # JSON.parse
-          body_parser.parse(sqs_msg.body)
-        elsif body_parser.respond_to?(:load)
-          # see https://github.com/phstc/shoryuken/pull/91
-          # JSON.load
-          body_parser.load(sqs_msg.body)
-        end
-      end
-    rescue => e
-      logger.error { "Error parsing the message body: #{e.message}\nbody_parser: #{body_parser}\nsqs_msg.body: #{sqs_msg.body}" }
-      raise
+    def worker_class
+      worker.class
+    end
+
+    def body
+      @_body ||= sqs_msg.is_a?(Array) ? sqs_msg.map(&method(:parse_body)) : parse_body(sqs_msg)
+    end
+
+    def parse_body(sqs_msg)
+      BodyParser.parse(worker_class, sqs_msg)
     end
   end
 end

data/lib/shoryuken/queue.rb

@@ -1,17 +1,16 @@
 module Shoryuken
   class Queue
-    FIFO_ATTR               = 'FifoQueue'
-    MESSAGE_GROUP_ID        = 'ShoryukenMessage'
-    VISIBILITY_TIMEOUT_ATTR = 'VisibilityTimeout'
+    include Util
+
+    FIFO_ATTR               = 'FifoQueue'.freeze
+    MESSAGE_GROUP_ID        = 'ShoryukenMessage'.freeze
+    VISIBILITY_TIMEOUT_ATTR = 'VisibilityTimeout'.freeze
 
     attr_accessor :name, :client, :url
 
-    def initialize(client, name)
-      self.name   = name
+    def initialize(client, name_or_url)
       self.client = client
-      self.url    = client.get_queue_url(queue_name: name).queue_url
-    rescue Aws::SQS::Errors::NonExistentQueue => e
-      raise e, "The specified queue '#{name}' does not exist."
+      set_name_and_url(name_or_url)
     end
 
     def visibility_timeout
@@ -19,7 +18,13 @@ module Shoryuken
     end
 
     def delete_messages(options)
-      client.delete_message_batch(options.merge(queue_url: url))
+      client.delete_message_batch(
+        options.merge(queue_url: url)
+      ).failed.any? do |failure|
+        logger.error do
+          "Could not delete #{failure.id}, code: '#{failure.code}', message: '#{failure.message}', sender_fault: #{failure.sender_fault}"
+        end
+      end
     end
 
     def send_message(options)
@@ -39,11 +44,37 @@ module Shoryuken
     end
 
     def fifo?
-      @_fifo ||= queue_attributes.attributes[FIFO_ATTR] == 'true'
+      # Make sure the memoization work with boolean to avoid multiple calls to SQS
+      # see https://github.com/phstc/shoryuken/pull/529
+      return @_fifo if defined?(@_fifo)
+      @_fifo = queue_attributes.attributes[FIFO_ATTR] == 'true'
     end
 
     private
 
+    def set_by_name(name)
+      self.name = name
+      self.url  = client.get_queue_url(queue_name: name).queue_url
+    end
+
+    def set_by_url(url)
+      self.name = url.split('/').last
+      self.url  = url
+    end
+
+    def set_name_and_url(name_or_url)
+      if name_or_url.start_with?('https://sqs.')
+        set_by_url(name_or_url)
+
+        # anticipate the fifo? checker for validating the queue URL
+        return fifo?
+      end
+
+      set_by_name(name_or_url)
+    rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue => ex
+      raise ex, "The specified queue #{name_or_url} does not exist."
+    end
+
     def queue_attributes
       # Note: Retrieving all queue attributes as requesting `FifoQueue` on non-FIFO queue raises error.
       # See issue: https://github.com/aws/aws-sdk-ruby/issues/1350

data/lib/shoryuken/runner.rb

@@ -8,7 +8,6 @@ require 'shoryuken'
 
 module Shoryuken
   # rubocop:disable Lint/InheritException
-  # rubocop:disable Metrics/AbcSize
   # See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
   class Shutdown < Interrupt; end
 
@@ -19,7 +18,7 @@ module Shoryuken
     def run(options)
       self_read, self_write = IO.pipe
 
-      %w(INT TERM USR1 USR2 TTIN).each do |sig|
+      %w[INT TERM USR1 TSTP TTIN].each do |sig|
         begin
           trap sig do
             self_write.puts(sig)
@@ -43,22 +42,15 @@ module Shoryuken
 
       @launcher = Shoryuken::Launcher.new
 
-      if (callback = Shoryuken.start_callback)
-        logger.info { 'Calling Shoryuken.on_start block' }
-        callback.call
-      end
-
-      fire_event(:startup)
-
       begin
-        @launcher.run
+        @launcher.start
 
         while (readable_io = IO.select([self_read]))
           signal = readable_io.first[0].gets.strip
           handle_signal(signal)
         end
       rescue Interrupt
-        @launcher.stop(shutdown: true)
+        @launcher.stop!
         exit 0
       end
     end
@@ -110,10 +102,15 @@ module Shoryuken
       logger.info { 'Received USR1, will soft shutdown down' }
 
       @launcher.stop
-      fire_event(:quiet, true)
       exit 0
     end
 
+    def execute_terminal_stop
+      logger.info { 'Received TSTP, will stop accepting new work' }
+
+      @launcher.stop
+    end
+
     def print_threads_backtrace
       Thread.list.each do |thread|
         logger.info { "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" }
@@ -126,15 +123,14 @@ module Shoryuken
     end
 
     def handle_signal(sig)
-      logger.info { "Got #{sig} signal" }
+      logger.debug "Got #{sig} signal"
 
       case sig
       when 'USR1' then execute_soft_shutdown
       when 'TTIN' then print_threads_backtrace
-      when 'USR2'
-        logger.warn { "Received #{sig}, will do nothing. To execute soft shutdown, please send USR1" }
-      else
-        logger.info { "Received #{sig}, will shutdown down" }
+      when 'TSTP' then execute_terminal_stop
+      when 'TERM', 'INT'
+        logger.info { "Received #{sig}, will shutdown" }
 
         raise Interrupt
       end

data/lib/shoryuken/util.rb

@@ -4,13 +4,13 @@ module Shoryuken
       Shoryuken.logger
     end
 
-    def fire_event(event, reverse = false)
+    def fire_event(event, reverse = false, event_options = {})
       logger.debug { "Firing '#{event}' lifecycle event" }
       arr = Shoryuken.options[:lifecycle_events][event]
       arr.reverse! if reverse
       arr.each do |block|
         begin
-          block.call
+          block.call(event_options)
         rescue => ex
           logger.warn(event: event)
           logger.warn "#{ex.class.name}: #{ex.message}"
@@ -30,7 +30,7 @@ module Shoryuken
     end
 
     def worker_name(worker_class, sqs_msg, body = nil)
-      if defined?(::ActiveJob) \
+      if Shoryuken.active_job? \
           && !sqs_msg.is_a?(Array) \
           && sqs_msg.message_attributes \
           && sqs_msg.message_attributes['shoryuken_class'] \