sidekiq 5.2.1 → 6.0.7

data/lib/sidekiq/job_retry.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
-require 'sidekiq/scheduled'
-require 'sidekiq/api'
+
+require "sidekiq/scheduled"
+require "sidekiq/api"
+
+require "zlib"
+require "base64"
 
 module Sidekiq
   ##
@@ -56,7 +60,8 @@ module Sidekiq
   #    end
   #
   class JobRetry
-    class Skip < ::RuntimeError; end
+    class Handled < ::RuntimeError; end
+    class Skip < Handled; end
 
     include Sidekiq::Util
 
@@ -69,9 +74,9 @@ module Sidekiq
     # The global retry handler requires only the barest of data.
     # We want to be able to retry as much as possible so we don't
     # require the worker to be instantiated.
-    def global(msg, queue)
+    def global(jobstr, queue)
       yield
-    rescue Skip => ex
+    rescue Handled => ex
       raise ex
     rescue Sidekiq::Shutdown => ey
       # ignore, will be pushed back onto queue during hard_shutdown
@@ -80,11 +85,19 @@ module Sidekiq
       # ignore, will be pushed back onto queue during hard_shutdown
       raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
 
-      raise e unless msg['retry']
-      attempt_retry(nil, msg, queue, e)
-      raise e
-    end
+      msg = Sidekiq.load_json(jobstr)
+      if msg["retry"]
+        attempt_retry(nil, msg, queue, e)
+      else
+        Sidekiq.death_handlers.each do |handler|
+          handler.call(msg, e)
+        rescue => handler_ex
+          handle_exception(handler_ex, {context: "Error calling death handler", job: msg})
+        end
+      end
 
+      raise Handled
+    end
 
     # The local retry support means that any errors that occur within
     # this block can be associated with the given worker instance.
@@ -94,9 +107,9 @@ module Sidekiq
     # exception so the global block does not reprocess the error.  The
     # Skip exception is unwrapped within Sidekiq::Processor#process before
     # calling the handle_exception handlers.
-    def local(worker, msg, queue)
+    def local(worker, jobstr, queue)
       yield
-    rescue Skip => ex
+    rescue Handled => ex
       raise ex
     rescue Sidekiq::Shutdown => ey
       # ignore, will be pushed back onto queue during hard_shutdown
@@ -105,11 +118,12 @@ module Sidekiq
       # ignore, will be pushed back onto queue during hard_shutdown
       raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
 
-      if msg['retry'] == nil
-        msg['retry'] = worker.class.get_sidekiq_options['retry']
+      msg = Sidekiq.load_json(jobstr)
+      if msg["retry"].nil?
+        msg["retry"] = worker.class.get_sidekiq_options["retry"]
       end
 
-      raise e unless msg['retry']
+      raise e unless msg["retry"]
       attempt_retry(worker, msg, queue, e)
       # We've handled this error associated with this job, don't
       # need to handle it at the global level
@@ -122,47 +136,44 @@ module Sidekiq
     # instantiate the worker instance.  All access must be guarded and
     # best effort.
     def attempt_retry(worker, msg, queue, exception)
-      max_retry_attempts = retry_attempts_from(msg['retry'], @max_retries)
+      max_retry_attempts = retry_attempts_from(msg["retry"], @max_retries)
 
-      msg['queue'] = if msg['retry_queue']
-        msg['retry_queue']
-      else
-        queue
-      end
+      msg["queue"] = (msg["retry_queue"] || queue)
 
-      # App code can stuff all sorts of crazy binary data into the error message
-      # that won't convert to JSON.
-      m = exception.message.to_s[0, 10_000]
+      m = exception_message(exception)
       if m.respond_to?(:scrub!)
         m.force_encoding("utf-8")
         m.scrub!
       end
 
-      msg['error_message'] = m
-      msg['error_class'] = exception.class.name
-      count = if msg['retry_count']
-        msg['retried_at'] = Time.now.to_f
-        msg['retry_count'] += 1
+      msg["error_message"] = m
+      msg["error_class"] = exception.class.name
+      count = if msg["retry_count"]
+        msg["retried_at"] = Time.now.to_f
+        msg["retry_count"] += 1
       else
-        msg['failed_at'] = Time.now.to_f
-        msg['retry_count'] = 0
+        msg["failed_at"] = Time.now.to_f
+        msg["retry_count"] = 0
       end
 
-      if msg['backtrace'] == true
-        msg['error_backtrace'] = exception.backtrace
-      elsif !msg['backtrace']
-        # do nothing
-      elsif msg['backtrace'].to_i != 0
-        msg['error_backtrace'] = exception.backtrace[0...msg['backtrace'].to_i]
+      if msg["backtrace"]
+        lines = if msg["backtrace"] == true
+          exception.backtrace
+        else
+          exception.backtrace[0...msg["backtrace"].to_i]
+        end
+
+        msg["error_backtrace"] = compress_backtrace(lines)
       end
 
       if count < max_retry_attempts
         delay = delay_for(worker, count, exception)
-        logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
+        # Logging here can break retries if the logging device raises ENOSPC #3979
+        # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
         retry_at = Time.now.to_f + delay
         payload = Sidekiq.dump_json(msg)
         Sidekiq.redis do |conn|
-          conn.zadd('retry', retry_at.to_s, payload)
+          conn.zadd("retry", retry_at.to_s, payload)
         end
       else
         # Goodbye dear message, you (re)tried your best I'm sure.
@@ -171,27 +182,24 @@ module Sidekiq
     end
 
     def retries_exhausted(worker, msg, exception)
-      logger.debug { "Retries exhausted for job" }
       begin
-        block = worker && worker.sidekiq_retries_exhausted_block
-        block.call(msg, exception) if block
+        block = worker&.sidekiq_retries_exhausted_block
+        block&.call(msg, exception)
       rescue => e
-        handle_exception(e, { context: "Error calling retries_exhausted", job: msg })
+        handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
       end
 
+      send_to_morgue(msg) unless msg["dead"] == false
+
       Sidekiq.death_handlers.each do |handler|
-        begin
-          handler.call(msg, exception)
-        rescue => e
-          handle_exception(e, { context: "Error calling death handler", job: msg })
-        end
+        handler.call(msg, exception)
+      rescue => e
+        handle_exception(e, {context: "Error calling death handler", job: msg})
       end
-
-      send_to_morgue(msg) unless msg['dead'] == false
     end
 
     def send_to_morgue(msg)
-      Sidekiq.logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
+      logger.info { "Adding dead #{msg["class"]} job #{msg["jid"]}" }
       payload = Sidekiq.dump_json(msg)
       DeadSet.new.kill(payload, notify_failure: false)
     end
@@ -205,7 +213,7 @@ module Sidekiq
     end
 
     def delay_for(worker, count, exception)
-      if worker && worker.sidekiq_retry_in_block
+      if worker&.sidekiq_retry_in_block
         custom_retry_in = retry_in(worker, count, exception).to_i
         return custom_retry_in if custom_retry_in > 0
       end
@@ -214,16 +222,14 @@ module Sidekiq
 
     # delayed_job uses the same basic formula
     def seconds_to_delay(count)
-      (count ** 4) + 15 + (rand(30)*(count+1))
+      (count**4) + 15 + (rand(30) * (count + 1))
     end
 
     def retry_in(worker, count, exception)
-      begin
-        worker.sidekiq_retry_in_block.call(count, exception)
-      rescue Exception => e
-        handle_exception(e, { context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
-        nil
-      end
+      worker.sidekiq_retry_in_block.call(count, exception)
+    rescue Exception => e
+      handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default"})
+      nil
     end
 
     def exception_caused_by_shutdown?(e, checked_causes = [])
@@ -237,5 +243,20 @@ module Sidekiq
         exception_caused_by_shutdown?(e.cause, checked_causes)
     end
 
+    # Extract message from exception.
+    # Set a default if the message raises an error
+    def exception_message(exception)
+      # App code can stuff all sorts of crazy binary data into the error message
+      # that won't convert to JSON.
+      exception.message.to_s[0, 10_000]
+    rescue
+      +"!!! ERROR MESSAGE THREW AN ERROR !!!"
+    end
+
+    def compress_backtrace(backtrace)
+      serialized = Sidekiq.dump_json(backtrace)
+      compressed = Zlib::Deflate.deflate(serialized)
+      Base64.encode64(compressed)
+    end
   end
 end

data/lib/sidekiq/launcher.rb

@@ -1,19 +1,25 @@
 # frozen_string_literal: true
-require 'sidekiq/manager'
-require 'sidekiq/fetch'
-require 'sidekiq/scheduled'
+
+require "sidekiq/manager"
+require "sidekiq/fetch"
+require "sidekiq/scheduled"
 
 module Sidekiq
-  # The Launcher is a very simple Actor whose job is to
-  # start, monitor and stop the core Actors in Sidekiq.
-  # If any of these actors die, the Sidekiq process exits
-  # immediately.
+  # The Launcher starts the Manager and Poller threads and provides the process heartbeat.
   class Launcher
     include Util
 
-    attr_accessor :manager, :poller, :fetcher
+    STATS_TTL = 5 * 365 * 24 * 60 * 60 # 5 years
 
-    STATS_TTL = 5*365*24*60*60
+    PROCTITLES = [
+      proc { "sidekiq" },
+      proc { Sidekiq::VERSION },
+      proc { |me, data| data["tag"] },
+      proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data["concurrency"]} busy]" },
+      proc { |me, data| "stopping" if me.stopping? }
+    ]
+
+    attr_accessor :manager, :poller, :fetcher
 
     def initialize(options)
       @manager = Sidekiq::Manager.new(options)
@@ -40,7 +46,7 @@ module Sidekiq
     # return until all work is complete and cleaned up.
     # It can take up to the timeout to complete.
     def stop
-      deadline = Time.now + @options[:timeout]
+      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @options[:timeout]
 
       @done = true
       @manager.quiet
@@ -62,17 +68,64 @@ module Sidekiq
 
     private unless $TESTING
 
+    def start_heartbeat
+      loop do
+        heartbeat
+        sleep 5
+      end
+      Sidekiq.logger.info("Heartbeat stopping...")
+    end
+
+    def clear_heartbeat
+      # Remove record from Redis since we are shutting down.
+      # Note we don't stop the heartbeat thread; if the process
+      # doesn't actually exit, it'll reappear in the Web UI.
+      Sidekiq.redis do |conn|
+        conn.pipelined do
+          conn.srem("processes", identity)
+          conn.unlink("#{identity}:workers")
+        end
+      end
+    rescue
+      # best effort, ignore network errors
+    end
+
     def heartbeat
-      results = Sidekiq::CLI::PROCTITLES.map {|x| x.(self, to_data) }
-      results.compact!
-      $0 = results.join(' ')
+      $0 = PROCTITLES.map { |proc| proc.call(self, to_data) }.compact.join(" ")
 
       ❤
     end
 
+    def self.flush_stats
+      fails = Processor::FAILURE.reset
+      procd = Processor::PROCESSED.reset
+      return if fails + procd == 0
+
+      nowdate = Time.now.utc.strftime("%Y-%m-%d")
+      begin
+        Sidekiq.redis do |conn|
+          conn.pipelined do
+            conn.incrby("stat:processed", procd)
+            conn.incrby("stat:processed:#{nowdate}", procd)
+            conn.expire("stat:processed:#{nowdate}", STATS_TTL)
+
+            conn.incrby("stat:failed", fails)
+            conn.incrby("stat:failed:#{nowdate}", fails)
+            conn.expire("stat:failed:#{nowdate}", STATS_TTL)
+          end
+        end
+      rescue => ex
+        # we're exiting the process, things might be shut down so don't
+        # try to handle the exception
+        Sidekiq.logger.warn("Unable to flush stats: #{ex}")
+      end
+    end
+    at_exit(&method(:flush_stats))
+
     def ❤
       key = identity
       fails = procd = 0
+
       begin
         fails = Processor::FAILURE.reset
         procd = Processor::PROCESSED.reset
@@ -80,6 +133,7 @@ module Sidekiq
 
         workers_key = "#{key}:workers"
         nowdate = Time.now.utc.strftime("%Y-%m-%d")
+
         Sidekiq.redis do |conn|
           conn.multi do
             conn.incrby("stat:processed", procd)
@@ -90,59 +144,52 @@ module Sidekiq
             conn.incrby("stat:failed:#{nowdate}", fails)
             conn.expire("stat:failed:#{nowdate}", STATS_TTL)
 
-            conn.del(workers_key)
+            conn.unlink(workers_key)
             curstate.each_pair do |tid, hash|
               conn.hset(workers_key, tid, Sidekiq.dump_json(hash))
             end
             conn.expire(workers_key, 60)
           end
         end
+
         fails = procd = 0
 
-        _, exists, _, _, msg = Sidekiq.redis do |conn|
-          conn.multi do
-            conn.sadd('processes', key)
+        _, exists, _, _, msg = Sidekiq.redis { |conn|
+          conn.multi {
+            conn.sadd("processes", key)
             conn.exists(key)
-            conn.hmset(key, 'info', to_json, 'busy', curstate.size, 'beat', Time.now.to_f, 'quiet', @done)
+            conn.hmset(key, "info", to_json, "busy", curstate.size, "beat", Time.now.to_f, "quiet", @done)
             conn.expire(key, 60)
             conn.rpop("#{key}-signals")
-          end
-        end
+          }
+        }
 
         # first heartbeat or recovering from an outage and need to reestablish our heartbeat
-        fire_event(:heartbeat) if !exists
+        fire_event(:heartbeat) unless exists
 
         return unless msg
 
-        ::Process.kill(msg, $$)
+        ::Process.kill(msg, ::Process.pid)
       rescue => e
         # ignore all redis/network issues
-        logger.error("heartbeat: #{e.message}")
+        logger.error("heartbeat: #{e}")
         # don't lose the counts if there was a network issue
         Processor::PROCESSED.incr(procd)
         Processor::FAILURE.incr(fails)
       end
     end
 
-    def start_heartbeat
-      while true
-        heartbeat
-        sleep 5
-      end
-      Sidekiq.logger.info("Heartbeat stopping...")
-    end
-
     def to_data
       @data ||= begin
         {
-          'hostname' => hostname,
-          'started_at' => Time.now.to_f,
-          'pid' => $$,
-          'tag' => @options[:tag] || '',
-          'concurrency' => @options[:concurrency],
-          'queues' => @options[:queues].uniq,
-          'labels' => @options[:labels],
-          'identity' => identity,
+          "hostname" => hostname,
+          "started_at" => Time.now.to_f,
+          "pid" => ::Process.pid,
+          "tag" => @options[:tag] || "",
+          "concurrency" => @options[:concurrency],
+          "queues" => @options[:queues].uniq,
+          "labels" => @options[:labels],
+          "identity" => identity
         }
       end
     end
@@ -154,20 +201,5 @@ module Sidekiq
         Sidekiq.dump_json(to_data)
       end
     end
-
-    def clear_heartbeat
-      # Remove record from Redis since we are shutting down.
-      # Note we don't stop the heartbeat thread; if the process
-      # doesn't actually exit, it'll reappear in the Web UI.
-      Sidekiq.redis do |conn|
-        conn.pipelined do
-          conn.srem('processes', identity)
-          conn.del("#{identity}:workers")
-        end
-      end
-    rescue
-      # best effort, ignore network errors
-    end
-
   end
 end

data/lib/sidekiq/logger.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require "logger"
+require "time"
+
+module Sidekiq
+  module Context
+    def self.with(hash)
+      current.merge!(hash)
+      yield
+    ensure
+      hash.each_key { |key| current.delete(key) }
+    end
+
+    def self.current
+      Thread.current[:sidekiq_context] ||= {}
+    end
+  end
+
+  module LoggingUtils
+    LEVELS = {
+      "debug" => 0,
+      "info" => 1,
+      "warn" => 2,
+      "error" => 3,
+      "fatal" => 4
+    }
+    LEVELS.default_proc = proc do |_, level|
+      Sidekiq.logger.warn("Invalid log level: #{level.inspect}")
+      nil
+    end
+
+    def debug?
+      level <= 0
+    end
+
+    def info?
+      level <= 1
+    end
+
+    def warn?
+      level <= 2
+    end
+
+    def error?
+      level <= 3
+    end
+
+    def fatal?
+      level <= 4
+    end
+
+    def local_level
+      Thread.current[:sidekiq_log_level]
+    end
+
+    def local_level=(level)
+      case level
+      when Integer
+        Thread.current[:sidekiq_log_level] = level
+      when Symbol, String
+        Thread.current[:sidekiq_log_level] = LEVELS[level.to_s]
+      when nil
+        Thread.current[:sidekiq_log_level] = nil
+      else
+        raise ArgumentError, "Invalid log level: #{level.inspect}"
+      end
+    end
+
+    def level
+      local_level || super
+    end
+
+    # Change the thread-local level for the duration of the given block.
+    def log_at(level)
+      old_local_level = local_level
+      self.local_level = level
+      yield
+    ensure
+      self.local_level = old_local_level
+    end
+
+    # Redefined to check severity against #level, and thus the thread-local level, rather than +@level+.
+    # FIXME: Remove when the minimum Ruby version supports overriding Logger#level.
+    def add(severity, message = nil, progname = nil, &block)
+      severity ||= ::Logger::UNKNOWN
+      progname ||= @progname
+
+      return true if @logdev.nil? || severity < level
+
+      if message.nil?
+        if block_given?
+          message = yield
+        else
+          message = progname
+          progname = @progname
+        end
+      end
+
+      @logdev.write format_message(format_severity(severity), Time.now, progname, message)
+    end
+  end
+
+  class Logger < ::Logger
+    include LoggingUtils
+
+    def initialize(*args, **kwargs)
+      super
+      self.formatter = Sidekiq.log_formatter
+    end
+
+    module Formatters
+      class Base < ::Logger::Formatter
+        def tid
+          Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
+        end
+
+        def ctx
+          Sidekiq::Context.current
+        end
+
+        def format_context
+          if ctx.any?
+            " " + ctx.compact.map { |k, v|
+              case v
+              when Array
+                "#{k}=#{v.join(",")}"
+              else
+                "#{k}=#{v}"
+              end
+            }.join(" ")
+          end
+        end
+      end
+
+      class Pretty < Base
+        def call(severity, time, program_name, message)
+          "#{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
+        end
+      end
+
+      class WithoutTimestamp < Pretty
+        def call(severity, time, program_name, message)
+          "pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
+        end
+      end
+
+      class JSON < Base
+        def call(severity, time, program_name, message)
+          hash = {
+            ts: time.utc.iso8601(3),
+            pid: ::Process.pid,
+            tid: tid,
+            lvl: severity,
+            msg: message
+          }
+          c = ctx
+          hash["ctx"] = c unless c.empty?
+
+          Sidekiq.dump_json(hash) << "\n"
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/manager.rb

@@ -1,12 +1,11 @@
 # frozen_string_literal: true
-require 'sidekiq/util'
-require 'sidekiq/processor'
-require 'sidekiq/fetch'
-require 'thread'
-require 'set'
 
-module Sidekiq
+require "sidekiq/util"
+require "sidekiq/processor"
+require "sidekiq/fetch"
+require "set"
 
+module Sidekiq
   ##
   # The Manager is the central coordination point in Sidekiq, controlling
   # the lifecycle of the Processors.
@@ -27,10 +26,10 @@ module Sidekiq
     attr_reader :workers
     attr_reader :options
 
-    def initialize(options={})
+    def initialize(options = {})
       logger.debug { options.inspect }
       @options = options
-      @count = options[:concurrency] || 25
+      @count = options[:concurrency] || 10
       raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
 
       @done = false
@@ -70,11 +69,11 @@ module Sidekiq
       return if @workers.empty?
 
       logger.info { "Pausing to allow workers to finish..." }
-      remaining = deadline - Time.now
+      remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       while remaining > PAUSE_TIME
         return if @workers.empty?
         sleep PAUSE_TIME
-        remaining = deadline - Time.now
+        remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
       return if @workers.empty?
 
@@ -113,7 +112,7 @@ module Sidekiq
       end
 
       if cleanup.size > 0
-        jobs = cleanup.map {|p| p.job }.compact
+        jobs = cleanup.map { |p| p.job }.compact
 
         logger.warn { "Terminating #{cleanup.size} busy worker threads" }
         logger.warn { "Work still in progress #{jobs.inspect}" }
@@ -132,6 +131,5 @@ module Sidekiq
         processor.kill
       end
     end
-
   end
 end