sidekiq 6.4.1 → 6.5.10

data/lib/sidekiq/job_retry.rb

@@ -1,10 +1,8 @@
 # frozen_string_literal: true
 
-require "sidekiq/scheduled"
-require "sidekiq/api"
-
 require "zlib"
 require "base64"
+require "sidekiq/component"
 
 module Sidekiq
   ##
@@ -25,11 +23,11 @@ module Sidekiq
   #
   # A job looks like:
   #
-  #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => true }
+  #     { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => true }
   #
   # The 'retry' option also accepts a number (in place of 'true'):
   #
-  #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => 5 }
+  #     { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => 5 }
   #
   # The job will be retried this number of times before giving up. (If simply
   # 'true', Sidekiq retries 25 times)
@@ -53,11 +51,11 @@ module Sidekiq
   #
   #   Sidekiq.options[:max_retries] = 7
   #
-  # or limit the number of retries for a particular worker and send retries to
+  # or limit the number of retries for a particular job and send retries to
   # a low priority queue with:
   #
-  #    class MyWorker
-  #      include Sidekiq::Worker
+  #    class MyJob
+  #      include Sidekiq::Job
   #      sidekiq_options retry: 10, retry_queue: 'low'
   #    end
   #
@@ -66,17 +64,18 @@ module Sidekiq
 
     class Skip < Handled; end
 
-    include Sidekiq::Util
+    include Sidekiq::Component
 
     DEFAULT_MAX_RETRY_ATTEMPTS = 25
 
-    def initialize(options = {})
-      @max_retries = Sidekiq.options.merge(options).fetch(:max_retries, DEFAULT_MAX_RETRY_ATTEMPTS)
+    def initialize(options)
+      @config = options
+      @max_retries = @config[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
     end
 
     # The global retry handler requires only the barest of data.
     # We want to be able to retry as much as possible so we don't
-    # require the worker to be instantiated.
+    # require the job to be instantiated.
     def global(jobstr, queue)
       yield
     rescue Handled => ex
@@ -90,7 +89,7 @@ module Sidekiq
 
       msg = Sidekiq.load_json(jobstr)
       if msg["retry"]
-        attempt_retry(nil, msg, queue, e)
+        process_retry(nil, msg, queue, e)
       else
         Sidekiq.death_handlers.each do |handler|
           handler.call(msg, e)
@@ -103,14 +102,14 @@ module Sidekiq
     end
 
     # The local retry support means that any errors that occur within
-    # this block can be associated with the given worker instance.
+    # this block can be associated with the given job instance.
     # This is required to support the `sidekiq_retries_exhausted` block.
     #
     # Note that any exception from the block is wrapped in the Skip
     # exception so the global block does not reprocess the error.  The
     # Skip exception is unwrapped within Sidekiq::Processor#process before
     # calling the handle_exception handlers.
-    def local(worker, jobstr, queue)
+    def local(jobinst, jobstr, queue)
       yield
     rescue Handled => ex
       raise ex
@@ -123,11 +122,11 @@ module Sidekiq
 
       msg = Sidekiq.load_json(jobstr)
       if msg["retry"].nil?
-        msg["retry"] = worker.class.get_sidekiq_options["retry"]
+        msg["retry"] = jobinst.class.get_sidekiq_options["retry"]
       end
 
       raise e unless msg["retry"]
-      attempt_retry(worker, msg, queue, e)
+      process_retry(jobinst, msg, queue, e)
       # We've handled this error associated with this job, don't
       # need to handle it at the global level
       raise Skip
@@ -135,10 +134,10 @@ module Sidekiq
 
     private
 
-    # Note that +worker+ can be nil here if an error is raised before we can
-    # instantiate the worker instance.  All access must be guarded and
+    # Note that +jobinst+ can be nil here if an error is raised before we can
+    # instantiate the job instance.  All access must be guarded and
     # best effort.
-    def attempt_retry(worker, msg, queue, exception)
+    def process_retry(jobinst, msg, queue, exception)
       max_retry_attempts = retry_attempts_from(msg["retry"], @max_retries)
 
       msg["queue"] = (msg["retry_queue"] || queue)
@@ -169,24 +168,54 @@ module Sidekiq
         msg["error_backtrace"] = compress_backtrace(lines)
       end
 
-      if count < max_retry_attempts
-        delay = delay_for(worker, count, exception)
-        # Logging here can break retries if the logging device raises ENOSPC #3979
-        # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
-        retry_at = Time.now.to_f + delay
-        payload = Sidekiq.dump_json(msg)
-        Sidekiq.redis do |conn|
-          conn.zadd("retry", retry_at.to_s, payload)
-        end
-      else
-        # Goodbye dear message, you (re)tried your best I'm sure.
-        retries_exhausted(worker, msg, exception)
+      # Goodbye dear message, you (re)tried your best I'm sure.
+      return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
+
+      strategy, delay = delay_for(jobinst, count, exception)
+      case strategy
+      when :discard
+        return # poof!
+      when :kill
+        return retries_exhausted(jobinst, msg, exception)
+      end
+
+      # Logging here can break retries if the logging device raises ENOSPC #3979
+      # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
+      jitter = rand(10) * (count + 1)
+      retry_at = Time.now.to_f + delay + jitter
+      payload = Sidekiq.dump_json(msg)
+      redis do |conn|
+        conn.zadd("retry", retry_at.to_s, payload)
+      end
+    end
+
+    # returns (strategy, seconds)
+    def delay_for(jobinst, count, exception)
+      rv = begin
+        # sidekiq_retry_in can return two different things:
+        # 1. When to retry next, as an integer of seconds
+        # 2. A symbol which re-routes the job elsewhere, e.g. :discard, :kill, :default
+        jobinst&.sidekiq_retry_in_block&.call(count, exception)
+      rescue Exception => e
+        handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
+        nil
+      end
+
+      delay = (count**4) + 15
+      if Integer === rv && rv > 0
+        delay = rv
+      elsif rv == :discard
+        return [:discard, nil] # do nothing, job goes poof
+      elsif rv == :kill
+        return [:kill, nil]
       end
+
+      [:default, delay]
     end
 
-    def retries_exhausted(worker, msg, exception)
+    def retries_exhausted(jobinst, msg, exception)
       begin
-        block = worker&.sidekiq_retries_exhausted_block
+        block = jobinst&.sidekiq_retries_exhausted_block
         block&.call(msg, exception)
       rescue => e
         handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
@@ -194,7 +223,7 @@ module Sidekiq
 
       send_to_morgue(msg) unless msg["dead"] == false
 
-      Sidekiq.death_handlers.each do |handler|
+      config.death_handlers.each do |handler|
         handler.call(msg, exception)
       rescue => e
         handle_exception(e, {context: "Error calling death handler", job: msg})
@@ -204,7 +233,15 @@ module Sidekiq
     def send_to_morgue(msg)
       logger.info { "Adding dead #{msg["class"]} job #{msg["jid"]}" }
       payload = Sidekiq.dump_json(msg)
-      DeadSet.new.kill(payload, notify_failure: false)
+      now = Time.now.to_f
+
+      config.redis do |conn|
+        conn.multi do |xa|
+          xa.zadd("dead", now.to_s, payload)
+          xa.zremrangebyscore("dead", "-inf", now - config[:dead_timeout_in_seconds])
+          xa.zremrangebyrank("dead", 0, - config[:dead_max_jobs])
+        end
+      end
     end
 
     def retry_attempts_from(msg_retry, default)
@@ -215,22 +252,6 @@ module Sidekiq
       end
     end
 
-    def delay_for(worker, count, exception)
-      jitter = rand(10) * (count + 1)
-      if worker&.sidekiq_retry_in_block
-        custom_retry_in = retry_in(worker, count, exception).to_i
-        return custom_retry_in + jitter if custom_retry_in > 0
-      end
-      (count**4) + 15 + jitter
-    end
-
-    def retry_in(worker, count, exception)
-      worker.sidekiq_retry_in_block.call(count, exception)
-    rescue Exception => e
-      handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default"})
-      nil
-    end
-
     def exception_caused_by_shutdown?(e, checked_causes = [])
       return false unless e.cause
 

data/lib/sidekiq/job_util.rb

@@ -4,7 +4,8 @@ require "time"
 module Sidekiq
   module JobUtil
     # These functions encapsulate various job utilities.
-    # They must be simple and free from side effects.
+
+    TRANSIENT_ATTRIBUTES = %w[]
 
     def validate(item)
       raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
@@ -12,16 +13,19 @@ module Sidekiq
       raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
       raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
       raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
+    end
 
-      if Sidekiq.options[:on_complex_arguments] == :raise
+    def verify_json(item)
+      job_class = item["wrapped"] || item["class"]
+      if Sidekiq[:on_complex_arguments] == :raise
         msg = <<~EOM
-          Job arguments to #{item["class"]} must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices.
+          Job arguments to #{job_class} must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices.
           To disable this error, remove `Sidekiq.strict_args!` from your initializer.
         EOM
         raise(ArgumentError, msg) unless json_safe?(item)
-      elsif Sidekiq.options[:on_complex_arguments] == :warn
+      elsif Sidekiq[:on_complex_arguments] == :warn
         Sidekiq.logger.warn <<~EOM unless json_safe?(item)
-          Job arguments to #{item["class"]} do not serialize to JSON safely. This will raise an error in
+          Job arguments to #{job_class} do not serialize to JSON safely. This will raise an error in
           Sidekiq 7.0. See https://github.com/mperham/sidekiq/wiki/Best-Practices or raise an error today
           by calling `Sidekiq.strict_args!` during Sidekiq initialization.
         EOM
@@ -39,20 +43,22 @@ module Sidekiq
 
       raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
 
+      # remove job attributes which aren't necessary to persist into Redis
+      TRANSIENT_ATTRIBUTES.each { |key| item.delete(key) }
+
+      item["jid"] ||= SecureRandom.hex(12)
       item["class"] = item["class"].to_s
       item["queue"] = item["queue"].to_s
-      item["jid"] ||= SecureRandom.hex(12)
       item["created_at"] ||= Time.now.to_f
-
       item
     end
 
     def normalized_hash(item_class)
       if item_class.is_a?(Class)
-        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?(:get_sidekiq_options)
+        raise(ArgumentError, "Message must include a Sidekiq::Job class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?(:get_sidekiq_options)
         item_class.get_sidekiq_options
       else
-        Sidekiq.default_worker_options
+        Sidekiq.default_job_options
       end
     end
 

data/lib/sidekiq/launcher.rb

@@ -3,11 +3,12 @@
 require "sidekiq/manager"
 require "sidekiq/fetch"
 require "sidekiq/scheduled"
+require "sidekiq/ring_buffer"
 
 module Sidekiq
   # The Launcher starts the Manager and Poller threads and provides the process heartbeat.
   class Launcher
-    include Util
+    include Sidekiq::Component
 
     STATS_TTL = 5 * 365 * 24 * 60 * 60 # 5 years
 
@@ -15,18 +16,18 @@ module Sidekiq
       proc { "sidekiq" },
       proc { Sidekiq::VERSION },
       proc { |me, data| data["tag"] },
-      proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data["concurrency"]} busy]" },
+      proc { |me, data| "[#{Processor::WORK_STATE.size} of #{data["concurrency"]} busy]" },
       proc { |me, data| "stopping" if me.stopping? }
     ]
 
     attr_accessor :manager, :poller, :fetcher
 
     def initialize(options)
+      @config = options
       options[:fetch] ||= BasicFetch.new(options)
       @manager = Sidekiq::Manager.new(options)
-      @poller = Sidekiq::Scheduled::Poller.new
+      @poller = Sidekiq::Scheduled::Poller.new(options)
       @done = false
-      @options = options
     end
 
     def run
@@ -43,11 +44,9 @@ module Sidekiq
       @poller.terminate
     end
 
-    # Shuts down the process.  This method does not
-    # return until all work is complete and cleaned up.
-    # It can take up to the timeout to complete.
+    # Shuts down this Sidekiq instance. Waits up to the deadline for all jobs to complete.
     def stop
-      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @options[:timeout]
+      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @config[:timeout]
 
       @done = true
       @manager.quiet
@@ -55,10 +54,10 @@ module Sidekiq
 
       @manager.stop(deadline)
 
-      # Requeue everything in case there was a worker who grabbed work while stopped
+      # Requeue everything in case there was a thread which fetched a job while the process was stopped.
       # This call is a no-op in Sidekiq but necessary for Sidekiq Pro.
-      strategy = @options[:fetch]
-      strategy.bulk_requeue([], @options)
+      strategy = @config[:fetch]
+      strategy.bulk_requeue([], @config)
 
       clear_heartbeat
     end
@@ -76,17 +75,19 @@ module Sidekiq
         heartbeat
         sleep BEAT_PAUSE
       end
-      Sidekiq.logger.info("Heartbeat stopping...")
+      logger.info("Heartbeat stopping...")
     end
 
     def clear_heartbeat
+      flush_stats
+
       # Remove record from Redis since we are shutting down.
       # Note we don't stop the heartbeat thread; if the process
       # doesn't actually exit, it'll reappear in the Web UI.
-      Sidekiq.redis do |conn|
+      redis do |conn|
         conn.pipelined do |pipeline|
-          pipeline.srem("processes", identity)
-          pipeline.unlink("#{identity}:workers")
+          pipeline.srem("processes", [identity])
+          pipeline.unlink("#{identity}:work")
         end
       end
     rescue
@@ -99,7 +100,7 @@ module Sidekiq
       ❤
     end
 
-    def self.flush_stats
+    def flush_stats
       fails = Processor::FAILURE.reset
       procd = Processor::PROCESSED.reset
       return if fails + procd == 0
@@ -123,7 +124,6 @@ module Sidekiq
         Sidekiq.logger.warn("Unable to flush stats: #{ex}")
       end
     end
-    at_exit(&method(:flush_stats))
 
     def ❤
       key = identity
@@ -132,12 +132,11 @@ module Sidekiq
       begin
         fails = Processor::FAILURE.reset
         procd = Processor::PROCESSED.reset
-        curstate = Processor::WORKER_STATE.dup
+        curstate = Processor::WORK_STATE.dup
 
-        workers_key = "#{key}:workers"
         nowdate = Time.now.utc.strftime("%Y-%m-%d")
 
-        Sidekiq.redis do |conn|
+        redis do |conn|
           conn.multi do |transaction|
             transaction.incrby("stat:processed", procd)
             transaction.incrby("stat:processed:#{nowdate}", procd)
@@ -146,12 +145,16 @@ module Sidekiq
             transaction.incrby("stat:failed", fails)
             transaction.incrby("stat:failed:#{nowdate}", fails)
             transaction.expire("stat:failed:#{nowdate}", STATS_TTL)
+          end
 
-            transaction.unlink(workers_key)
+          # work is the current set of executing jobs
+          work_key = "#{key}:work"
+          conn.pipelined do |transaction|
+            transaction.unlink(work_key)
             curstate.each_pair do |tid, hash|
-              transaction.hset(workers_key, tid, Sidekiq.dump_json(hash))
+              transaction.hset(work_key, tid, Sidekiq.dump_json(hash))
             end
-            transaction.expire(workers_key, 60)
+            transaction.expire(work_key, 60)
           end
         end
 
@@ -160,15 +163,15 @@ module Sidekiq
         fails = procd = 0
         kb = memory_usage(::Process.pid)
 
-        _, exists, _, _, msg = Sidekiq.redis { |conn|
+        _, exists, _, _, msg = redis { |conn|
           conn.multi { |transaction|
-            transaction.sadd("processes", key)
+            transaction.sadd("processes", [key])
             transaction.exists?(key)
             transaction.hmset(key, "info", to_json,
               "busy", curstate.size,
               "beat", Time.now.to_f,
               "rtt_us", rtt,
-              "quiet", @done,
+              "quiet", @done.to_s,
               "rss", kb)
             transaction.expire(key, 60)
             transaction.rpop("#{key}-signals")
@@ -177,6 +180,7 @@ module Sidekiq
 
         # first heartbeat or recovering from an outage and need to reestablish our heartbeat
         fire_event(:heartbeat) unless exists
+        fire_event(:beat, oneshot: false)
 
         return unless msg
 
@@ -198,7 +202,7 @@ module Sidekiq
 
     def check_rtt
       a = b = 0
-      Sidekiq.redis do |x|
+      redis do |x|
         a = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
         x.ping
         b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
@@ -209,12 +213,12 @@ module Sidekiq
       # Workable is < 10,000µs
       # Log a warning if it's a disaster.
       if RTT_READINGS.all? { |x| x > RTT_WARNING_LEVEL }
-        Sidekiq.logger.warn <<~EOM
+        logger.warn <<~EOM
           Your Redis network connection is performing extremely poorly.
           Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
           Ensure Redis is running in the same AZ or datacenter as Sidekiq.
           If these values are close to 100,000, that means your Sidekiq process may be
-          CPU overloaded; see https://github.com/mperham/sidekiq/discussions/5039
+          CPU-saturated; reduce your concurrency and/or see https://github.com/mperham/sidekiq/discussions/5039
         EOM
         RTT_READINGS.reset
       end
@@ -246,10 +250,10 @@ module Sidekiq
         "hostname" => hostname,
         "started_at" => Time.now.to_f,
         "pid" => ::Process.pid,
-        "tag" => @options[:tag] || "",
-        "concurrency" => @options[:concurrency],
-        "queues" => @options[:queues].uniq,
-        "labels" => @options[:labels],
+        "tag" => @config[:tag] || "",
+        "concurrency" => @config[:concurrency],
+        "queues" => @config[:queues].uniq,
+        "labels" => @config[:labels],
         "identity" => identity
       }
     end

data/lib/sidekiq/logger.rb

@@ -18,7 +18,7 @@ module Sidekiq
     end
 
     def self.add(k, v)
-      Thread.current[:sidekiq_context][k] = v
+      current[k] = v
     end
   end
 
@@ -35,24 +35,10 @@ module Sidekiq
       nil
     end
 
-    def debug?
-      level <= 0
-    end
-
-    def info?
-      level <= 1
-    end
-
-    def warn?
-      level <= 2
-    end
-
-    def error?
-      level <= 3
-    end
-
-    def fatal?
-      level <= 4
+    LEVELS.each do |level, numeric_level|
+      define_method("#{level}?") do
+        local_level.nil? ? super() : local_level <= numeric_level
+      end
     end
 
     def local_level

data/lib/sidekiq/manager.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "sidekiq/util"
 require "sidekiq/processor"
 require "sidekiq/fetch"
 require "set"
@@ -21,37 +20,34 @@ module Sidekiq
   # the shutdown process.  The other tasks are performed by other threads.
   #
   class Manager
-    include Util
+    include Sidekiq::Component
 
     attr_reader :workers
-    attr_reader :options
 
     def initialize(options = {})
+      @config = options
       logger.debug { options.inspect }
-      @options = options
       @count = options[:concurrency] || 10
       raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
 
       @done = false
       @workers = Set.new
       @count.times do
-        @workers << Processor.new(self, options)
+        @workers << Processor.new(@config, &method(:processor_result))
       end
       @plock = Mutex.new
     end
 
     def start
-      @workers.each do |x|
-        x.start
-      end
+      @workers.each(&:start)
     end
 
     def quiet
       return if @done
       @done = true
 
-      logger.info { "Terminating quiet workers" }
-      @workers.each { |x| x.terminate }
+      logger.info { "Terminating quiet threads" }
+      @workers.each(&:terminate)
       fire_event(:quiet, reverse: true)
     end
 
@@ -65,24 +61,18 @@ module Sidekiq
       sleep PAUSE_TIME
       return if @workers.empty?
 
-      logger.info { "Pausing to allow workers to finish..." }
+      logger.info { "Pausing to allow jobs to finish..." }
       wait_for(deadline) { @workers.empty? }
       return if @workers.empty?
 
       hard_shutdown
     end
 
-    def processor_stopped(processor)
-      @plock.synchronize do
-        @workers.delete(processor)
-      end
-    end
-
-    def processor_died(processor, reason)
+    def processor_result(processor, reason = nil)
       @plock.synchronize do
         @workers.delete(processor)
         unless @done
-          p = Processor.new(self, options)
+          p = Processor.new(@config, &method(:processor_result))
           @workers << p
           p.start
         end
@@ -96,7 +86,7 @@ module Sidekiq
     private
 
     def hard_shutdown
-      # We've reached the timeout and we still have busy workers.
+      # We've reached the timeout and we still have busy threads.
       # They must die but their jobs shall live on.
       cleanup = nil
       @plock.synchronize do
@@ -106,17 +96,17 @@ module Sidekiq
       if cleanup.size > 0
         jobs = cleanup.map { |p| p.job }.compact
 
-        logger.warn { "Terminating #{cleanup.size} busy worker threads" }
-        logger.warn { "Work still in progress #{jobs.inspect}" }
+        logger.warn { "Terminating #{cleanup.size} busy threads" }
+        logger.debug { "Jobs still in progress #{jobs.inspect}" }
 
         # Re-enqueue unfinished jobs
         # NOTE: You may notice that we may push a job back to redis before
-        # the worker thread is terminated. This is ok because Sidekiq's
+        # the thread is terminated. This is ok because Sidekiq's
         # contract says that jobs are run AT LEAST once. Process termination
         # is delayed until we're certain the jobs are back in Redis because
         # it is worse to lose a job than to run it twice.
-        strategy = @options[:fetch]
-        strategy.bulk_requeue(jobs, @options)
+        strategy = @config[:fetch]
+        strategy.bulk_requeue(jobs, @config)
       end
 
       cleanup.each do |processor|
@@ -129,5 +119,18 @@ module Sidekiq
       deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + 3
       wait_for(deadline) { @workers.empty? }
     end
+
+    # hack for quicker development / testing environment #2774
+    PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
+
+    # Wait for the orblock to be true or the deadline passed.
+    def wait_for(deadline, &condblock)
+      remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      while remaining > PAUSE_TIME
+        return if condblock.call
+        sleep PAUSE_TIME
+        remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
   end
 end