shoryuken 2.1.0 → 3.0.0

data/lib/shoryuken/middleware/server/auto_extend_visibility.rb

@@ -1,33 +1,32 @@
-require 'celluloid/current' unless defined?(Celluloid)
-
 module Shoryuken
   module Middleware
     module Server
       class AutoExtendVisibility
+        include Util
+
         EXTEND_UPFRONT_SECONDS = 5
 
         def call(worker, queue, sqs_msg, body)
-          timer = auto_visibility_timer(worker, queue, sqs_msg, body)
-          begin
-            yield
-          ensure
-            if timer
-              timer.cancel
-              @visibility_extender.terminate
-            end
+          if sqs_msg.is_a?(Array)
+            logger.warn { "Auto extend visibility isn't supported for batch workers" }
+            return yield
           end
+
+          timer = auto_visibility_timer(worker, queue, sqs_msg, body)
+          yield
+        ensure
+          timer.kill if timer
         end
 
         private
 
         class MessageVisibilityExtender
-          include Celluloid
           include Util
 
           def auto_extend(worker, queue, sqs_msg, body)
             queue_visibility_timeout = Shoryuken::Client.queues(queue).visibility_timeout
 
-            every(queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
+            Concurrent::TimerTask.new(execution_interval: queue_visibility_timeout - EXTEND_UPFRONT_SECONDS) do
               begin
                 logger.debug do
                   "Extending message #{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.message_id} " \
@@ -48,8 +47,8 @@ module Shoryuken
 
         def auto_visibility_timer(worker, queue, sqs_msg, body)
           return unless worker.class.auto_visibility_timeout?
-          @visibility_extender = MessageVisibilityExtender.new_link
-          @visibility_extender.auto_extend(worker, queue, sqs_msg, body)
+
+          MessageVisibilityExtender.new.auto_extend(worker, queue, sqs_msg, body).tap(&:execute)
         end
       end
     end

data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb

@@ -5,12 +5,17 @@ module Shoryuken
         include Util
 
         def call(worker, queue, sqs_msg, body)
+          if sqs_msg.is_a?(Array)
+            logger.warn { "Exponential backoff isn't supported for batch workers" }
+            return yield
+          end
+
           started_at = Time.now
           yield
         rescue
-          retry_intervals = Array(worker.class.get_shoryuken_options['retry_intervals'])
+          retry_intervals = worker.class.get_shoryuken_options['retry_intervals']
 
-          if retry_intervals.empty? || !handle_failure(sqs_msg, started_at, retry_intervals)
+          if retry_intervals.nil? || !handle_failure(sqs_msg, started_at, retry_intervals)
             # Re-raise the exception if the job is not going to be exponential backoff retried.
             # This allows custom middleware (like exception notifiers) to be aware of the unhandled failure.
             raise
@@ -19,28 +24,32 @@ module Shoryuken
 
         private
 
-        def handle_failure(sqs_msg, started_at, retry_intervals)
-          return unless attempts = sqs_msg.attributes['ApproximateReceiveCount']
-
-          attempts = attempts.to_i - 1
-
-          interval = if attempts < retry_intervals.size
-                       retry_intervals[attempts]
-                     else
-                       retry_intervals.last
-                     end
-
-          # Visibility timeouts are limited to a total 12 hours, starting from the receipt of the message.
-          # We calculate the maximum timeout by subtracting the amount of time since the receipt of the message.
-          #
-          # From the docs:  "Amazon SQS restarts the timeout period using the new value."
-          # http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/AboutVT.html#AboutVT-extending-message-visibility-timeout
-          max_timeout = 43200 - (Time.now - started_at).ceil - 1
+        def get_interval(retry_intervals, attempts)
+          return retry_intervals.call(attempts) if retry_intervals.respond_to?(:call)
+
+          if attempts <= (retry_intervals = Array(retry_intervals)).size
+            retry_intervals[attempts - 1]
+          else
+            retry_intervals.last
+          end
+        end
+
+        def next_visibility_timeout(interval, started_at)
+          max_timeout = 43_200 - (Time.now - started_at).ceil - 1
           interval = max_timeout if interval > max_timeout
+          interval.to_i
+        end
+
+        def handle_failure(sqs_msg, started_at, retry_intervals)
+          receive_count = sqs_msg.attributes['ApproximateReceiveCount'].to_i
+
+          return false unless (interval = get_interval(retry_intervals, receive_count))
 
-          sqs_msg.change_visibility(visibility_timeout: interval.to_i)
+          sqs_msg.change_visibility(visibility_timeout: next_visibility_timeout(interval.to_i, started_at))
 
           logger.info { "Message #{sqs_msg.message_id} failed, will be retried in #{interval} seconds." }
+
+          true
         end
       end
     end

data/lib/shoryuken/polling.rb

@@ -0,0 +1,204 @@
+module Shoryuken
+  module Polling
+    QueueConfiguration = Struct.new(:name, :options) do
+      def hash
+        name.hash
+      end
+
+      def ==(other)
+        case other
+        when String
+          if options.empty?
+            name == other
+          else
+            false
+          end
+        else
+          super
+        end
+      end
+
+      alias_method :eql?, :==
+
+      def to_s
+        if options.empty?
+          name
+        else
+          "#<QueueConfiguration #{name} options=#{options.inspect}>"
+        end
+      end
+    end
+
+    class BaseStrategy
+      include Util
+
+      def next_queue
+        fail NotImplementedError
+      end
+
+      def messages_found(queue, messages_found)
+        fail NotImplementedError
+      end
+
+      def active_queues
+        fail NotImplementedError
+      end
+
+      def ==(other)
+        case other
+        when Array
+          @queues == other
+        else
+          if other.respond_to?(:active_queues)
+            active_queues == other.active_queues
+          else
+            false
+          end
+        end
+      end
+
+      private
+
+      def delay
+        Shoryuken.options[:delay].to_f
+      end
+    end
+
+    class WeightedRoundRobin < BaseStrategy
+      def initialize(queues)
+        @initial_queues = queues
+        @queues = queues.dup.uniq
+        @paused_queues = []
+      end
+
+      def next_queue
+        unpause_queues
+        queue = @queues.shift
+        return nil if queue.nil?
+
+        @queues << queue
+        QueueConfiguration.new(queue, {})
+      end
+
+      def messages_found(queue, messages_found)
+        if messages_found == 0
+          pause(queue)
+          return
+        end
+
+        maximum_weight = maximum_queue_weight(queue)
+        current_weight = current_queue_weight(queue)
+        if maximum_weight > current_weight
+          logger.info { "Increasing '#{queue}' weight to #{current_weight + 1}, max: #{maximum_weight}" }
+          @queues << queue
+        end
+      end
+
+      def active_queues
+        unparse_queues(@queues)
+      end
+
+      private
+
+      def pause(queue)
+        return unless @queues.delete(queue)
+        @paused_queues << [Time.now + delay, queue]
+        logger.debug "Paused '#{queue}'"
+      end
+
+      def unpause_queues
+        return if @paused_queues.empty?
+        return if Time.now < @paused_queues.first[0]
+        pause = @paused_queues.shift
+        @queues << pause[1]
+        logger.debug "Unpaused '#{pause[1]}'"
+      end
+
+      def current_queue_weight(queue)
+        queue_weight(@queues, queue)
+      end
+
+      def maximum_queue_weight(queue)
+        queue_weight(@initial_queues, queue)
+      end
+
+      def queue_weight(queues, queue)
+        queues.count { |q| q == queue }
+      end
+    end
+
+    class StrictPriority < BaseStrategy
+      def initialize(queues)
+        # Priority ordering of the queues, highest priority first
+        @queues = queues
+          .group_by { |q| q }
+          .sort_by { |_, qs| -qs.count }
+          .map(&:first)
+
+        # Pause status of the queues, default to past time (unpaused)
+        @paused_until = queues
+          .each_with_object(Hash.new) { |queue, h| h[queue] = Time.at(0) }
+
+        # Start queues at 0
+        reset_next_queue
+      end
+
+      def next_queue
+        next_queue = next_active_queue
+        next_queue.nil? ? nil : QueueConfiguration.new(next_queue, {})
+      end
+
+      def messages_found(queue, messages_found)
+        if messages_found == 0
+          pause(queue)
+        else
+          reset_next_queue
+        end
+      end
+
+      def active_queues
+        @queues
+          .reverse
+          .map.with_index(1)
+          .reject { |q, _| queue_paused?(q) }
+          .reverse
+      end
+
+      private
+
+      def next_active_queue
+        reset_next_queue if queues_unpaused_since?
+
+        size = @queues.length
+        size.times do
+          queue = @queues[@next_queue_index]
+          @next_queue_index = (@next_queue_index + 1) % size
+          return queue unless queue_paused?(queue)
+        end
+
+        nil
+      end
+
+      def queues_unpaused_since?
+        last = @last_unpause_check
+        now = @last_unpause_check = Time.now
+
+        last && @paused_until.values.any? { |t| t > last && t <= now }
+      end
+
+      def reset_next_queue
+        @next_queue_index = 0
+      end
+
+      def queue_paused?(queue)
+        @paused_until[queue] > Time.now
+      end
+
+      def pause(queue)
+        return unless delay > 0
+        @paused_until[queue] = Time.now + delay
+        logger.debug "Paused '#{queue}'"
+      end
+    end
+  end
+end

data/lib/shoryuken/processor.rb

@@ -1,28 +1,20 @@
-require 'json'
-
 module Shoryuken
   class Processor
-    include Celluloid
     include Util
 
     def initialize(manager)
       @manager = manager
     end
 
-    attr_accessor :proxy_id
-
     def process(queue, sqs_msg)
-      @manager.async.real_thread(proxy_id, Thread.current)
-
       worker = Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
+      body = get_body(worker.class, sqs_msg)
 
-        body = get_body(worker.class, sqs_msg)
-
-        worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
-          worker.perform(sqs_msg, body)
-        end
-
-        @manager.async.processor_done(queue, current_actor)
+      worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
+        worker.perform(sqs_msg, body)
+      end
+    ensure
+      @manager.processor_done(queue)
     end
 
     private

data/lib/shoryuken/queue.rb

@@ -35,9 +35,7 @@ module Shoryuken
     end
 
     def receive_messages(options)
-      client.receive_message(options.merge(queue_url: url)).
-        messages.
-        map { |m| Message.new(client, self, m) }
+      client.receive_message(options.merge(queue_url: url)).messages.map { |m| Message.new(client, self, m) }
     end
 
     def fifo?

data/lib/shoryuken/runner.rb

@@ -0,0 +1,143 @@
+$stdout.sync = true
+
+require 'singleton'
+require 'optparse'
+require 'erb'
+
+require 'shoryuken'
+
+module Shoryuken
+  # rubocop:disable Lint/InheritException
+  # rubocop:disable Metrics/AbcSize
+  # See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
+  class Shutdown < Interrupt; end
+
+  class Runner
+    include Util
+    include Singleton
+
+    def run(options)
+      self_read, self_write = IO.pipe
+
+      %w(INT TERM USR1 USR2 TTIN).each do |sig|
+        begin
+          trap sig do
+            self_write.puts(sig)
+          end
+        rescue ArgumentError
+          puts "Signal #{sig} not supported"
+        end
+      end
+
+      loader = EnvironmentLoader.setup_options(options)
+
+      # When cli args exist, override options in config file
+      Shoryuken.options.merge!(options)
+
+      daemonize(Shoryuken.options)
+      write_pid(Shoryuken.options)
+
+      loader.load
+
+      initialize_concurrent_logger
+
+      @launcher = Shoryuken::Launcher.new
+
+      if (callback = Shoryuken.start_callback)
+        logger.info { 'Calling Shoryuken.on_start block' }
+        callback.call
+      end
+
+      fire_event(:startup)
+
+      begin
+        @launcher.run
+
+        while (readable_io = IO.select([self_read]))
+          signal = readable_io.first[0].gets.strip
+          handle_signal(signal)
+        end
+      rescue Interrupt
+        @launcher.stop(shutdown: true)
+        exit 0
+      end
+    end
+
+    private
+
+    def initialize_concurrent_logger
+      return unless Shoryuken.logger
+
+      Concurrent.global_logger = lambda do |level, progname, msg = nil, &block|
+        Shoryuken.logger.log(level, msg, progname, &block)
+      end
+    end
+
+    def daemonize(options)
+      return unless options[:daemon]
+
+      files_to_reopen = []
+      ObjectSpace.each_object(File) do |file|
+        files_to_reopen << file unless file.closed?
+      end
+
+      Process.daemon(true, true)
+
+      files_to_reopen.each do |file|
+        begin
+          file.reopen file.path, 'a+'
+          file.sync = true
+        rescue ::Exception
+        end
+      end
+
+      [$stdout, $stderr].each do |io|
+        File.open(options[:logfile], 'ab') do |f|
+          io.reopen(f)
+        end
+        io.sync = true
+      end
+      $stdin.reopen('/dev/null')
+    end
+
+    def write_pid(options)
+      return unless (path = options[:pidfile])
+
+      File.open(path, 'w') { |f| f.puts(Process.pid) }
+    end
+
+    def execute_soft_shutdown
+      logger.info { 'Received USR1, will soft shutdown down' }
+
+      @launcher.stop
+      fire_event(:quiet, true)
+      exit 0
+    end
+
+    def print_threads_backtrace
+      Thread.list.each do |thread|
+        logger.info { "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" }
+        if thread.backtrace
+          logger.info { thread.backtrace.join("\n") }
+        else
+          logger.info { '<no backtrace available>' }
+        end
+      end
+    end
+
+    def handle_signal(sig)
+      logger.info { "Got #{sig} signal" }
+
+      case sig
+      when 'USR1' then execute_soft_shutdown
+      when 'TTIN' then print_threads_backtrace
+      when 'USR2'
+        logger.warn { "Received #{sig}, will do nothing. To execute soft shutdown, please send USR1" }
+      else
+        logger.info { "Received #{sig}, will shutdown down" }
+
+        raise Interrupt
+      end
+    end
+  end
+end

data/lib/shoryuken/util.rb

@@ -1,13 +1,5 @@
 module Shoryuken
   module Util
-    def watchdog(last_words)
-      yield
-    rescue => ex
-      logger.error { last_words }
-      logger.error { ex }
-      logger.error { ex.backtrace.join("\n") }
-    end
-
     def logger
       Shoryuken.logger
     end

data/lib/shoryuken/version.rb

@@ -1,3 +1,3 @@
 module Shoryuken
-  VERSION = '2.1.0'
+  VERSION = '3.0.0'.freeze
 end

data/lib/shoryuken/worker.rb

@@ -63,7 +63,7 @@ module Shoryuken
 
       def normalize_worker_queue!
         queue = @shoryuken_options['queue']
-        if queue.respond_to? :call
+        if queue.respond_to?(:call)
           queue = queue.call
           @shoryuken_options['queue'] = queue
         end