sidekiq 5.2.10 → 6.5.12

data/lib/sidekiq/fetch.rb

@@ -1,81 +1,91 @@
 # frozen_string_literal: true
-require 'sidekiq'
 
-module Sidekiq
+require "sidekiq"
+require "sidekiq/component"
+
+module Sidekiq # :nodoc:
   class BasicFetch
+    include Sidekiq::Component
     # We want the fetch operation to timeout every few seconds so the thread
     # can check if the process is shutting down.
     TIMEOUT = 2
 
-    UnitOfWork = Struct.new(:queue, :job) do
+    UnitOfWork = Struct.new(:queue, :job, :config) {
       def acknowledge
         # nothing to do
       end
 
       def queue_name
-        queue.sub(/.*queue:/, '')
+        queue.delete_prefix("queue:")
       end
 
       def requeue
-        Sidekiq.redis do |conn|
-          conn.rpush("queue:#{queue_name}", job)
+        config.redis do |conn|
+          conn.rpush(queue, job)
         end
       end
-    end
+    }
 
-    def initialize(options)
-      @strictly_ordered_queues = !!options[:strict]
-      @queues = options[:queues].map { |q| "queue:#{q}" }
+    def initialize(config)
+      raise ArgumentError, "missing queue list" unless config[:queues]
+      @config = config
+      @strictly_ordered_queues = !!@config[:strict]
+      @queues = @config[:queues].map { |q| "queue:#{q}" }
       if @strictly_ordered_queues
-        @queues = @queues.uniq
-        @queues << TIMEOUT
+        @queues.uniq!
+        @queues << {timeout: TIMEOUT}
       end
     end
 
     def retrieve_work
-      work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
-      UnitOfWork.new(*work) if work
-    end
-
-    # Creating the Redis#brpop command takes into account any
-    # configured queue weights. By default Redis#brpop returns
-    # data from the first queue that has pending elements. We
-    # recreate the queue command each time we invoke Redis#brpop
-    # to honor weights and avoid queue starvation.
-    def queues_cmd
-      if @strictly_ordered_queues
-        @queues
-      else
-        queues = @queues.shuffle.uniq
-        queues << TIMEOUT
-        queues
+      qs = queues_cmd
+      # 4825 Sidekiq Pro with all queues paused will return an
+      # empty set of queues with a trailing TIMEOUT value.
+      if qs.size <= 1
+        sleep(TIMEOUT)
+        return nil
       end
-    end
 
+      queue, job = redis { |conn| conn.brpop(*qs) }
+      UnitOfWork.new(queue, job, config) if queue
+    end
 
-    # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
-    # an instance method will make it async to the Fetcher actor
-    def self.bulk_requeue(inprogress, options)
+    def bulk_requeue(inprogress, options)
       return if inprogress.empty?
 
-      Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+      logger.debug { "Re-queueing terminated jobs" }
       jobs_to_requeue = {}
       inprogress.each do |unit_of_work|
-        jobs_to_requeue[unit_of_work.queue_name] ||= []
-        jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.job
+        jobs_to_requeue[unit_of_work.queue] ||= []
+        jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
       end
 
-      Sidekiq.redis do |conn|
-        conn.pipelined do
+      redis do |conn|
+        conn.pipelined do |pipeline|
           jobs_to_requeue.each do |queue, jobs|
-            conn.rpush("queue:#{queue}", jobs)
+            pipeline.rpush(queue, jobs)
           end
         end
       end
-      Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
+      logger.info("Pushed #{inprogress.size} jobs back to Redis")
     rescue => ex
-      Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
+      logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
     end
 
+    # Creating the Redis#brpop command takes into account any
+    # configured queue weights. By default Redis#brpop returns
+    # data from the first queue that has pending elements. We
+    # recreate the queue command each time we invoke Redis#brpop
+    # to honor weights and avoid queue starvation.
+    def queues_cmd
+      if @strictly_ordered_queues
+        @queues
+      else
+        permute = @queues.shuffle
+        permute.uniq!
+        permute << {timeout: TIMEOUT}
+        permute
+      end
+    end
   end
 end

data/lib/sidekiq/job.rb

@@ -0,0 +1,13 @@
+require "sidekiq/worker"
+
+module Sidekiq
+  # Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
+  # Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
+  #
+  # The term "worker" is too generic and overly confusing, used in several
+  # different contexts meaning different things. Many people call a Sidekiq
+  # process a "worker". Some people call the thread that executes jobs a
+  # "worker". This change brings Sidekiq closer to ActiveJob where your job
+  # classes extend ApplicationJob.
+  Job = Worker
+end

data/lib/sidekiq/job_logger.rb

@@ -1,25 +1,51 @@
 # frozen_string_literal: true
+
 module Sidekiq
   class JobLogger
+    def initialize(logger = Sidekiq.logger)
+      @logger = logger
+    end
 
     def call(item, queue)
       start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      logger.info("start")
+      @logger.info("start")
+
       yield
-      logger.info("done: #{elapsed(start)} sec")
+
+      Sidekiq::Context.add(:elapsed, elapsed(start))
+      @logger.info("done")
     rescue Exception
-      logger.info("fail: #{elapsed(start)} sec")
+      Sidekiq::Context.add(:elapsed, elapsed(start))
+      @logger.info("fail")
+
       raise
     end
 
+    def prepare(job_hash, &block)
+      # If we're using a wrapper class, like ActiveJob, use the "wrapped"
+      # attribute to expose the underlying thing.
+      h = {
+        class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
+        jid: job_hash["jid"]
+      }
+      h[:bid] = job_hash["bid"] if job_hash.has_key?("bid")
+      h[:tags] = job_hash["tags"] if job_hash.has_key?("tags")
+
+      Thread.current[:sidekiq_context] = h
+      level = job_hash["log_level"]
+      if level
+        @logger.log_at(level, &block)
+      else
+        yield
+      end
+    ensure
+      Thread.current[:sidekiq_context] = nil
+    end
+
     private
 
     def elapsed(start)
       (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start).round(3)
     end
-
-    def logger
-      Sidekiq.logger
-    end
   end
 end

data/lib/sidekiq/job_retry.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
-require 'sidekiq/scheduled'
-require 'sidekiq/api'
+
+require "zlib"
+require "base64"
+require "sidekiq/component"
 
 module Sidekiq
   ##
@@ -21,18 +23,19 @@ module Sidekiq
   #
   # A job looks like:
   #
-  #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => true }
+  #     { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => true }
   #
   # The 'retry' option also accepts a number (in place of 'true'):
   #
-  #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => 5 }
+  #     { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => 5 }
   #
   # The job will be retried this number of times before giving up. (If simply
   # 'true', Sidekiq retries 25 times)
   #
-  # We'll add a bit more data to the job to support retries:
+  # Relevant options for job retries:
   #
-  #  * 'queue' - the queue to use
+  #  * 'queue' - the queue for the initial job
+  #  * 'retry_queue' - if job retries should be pushed to a different (e.g. lower priority) queue
   #  * 'retry_count' - number of times we've retried so far.
   #  * 'error_message' - the message from the exception
   #  * 'error_class' - the exception class
@@ -48,29 +51,32 @@ module Sidekiq
   #
   #   Sidekiq.options[:max_retries] = 7
   #
-  # or limit the number of retries for a particular worker with:
+  # or limit the number of retries for a particular job and send retries to
+  # a low priority queue with:
   #
-  #    class MyWorker
-  #      include Sidekiq::Worker
-  #      sidekiq_options :retry => 10
+  #    class MyJob
+  #      include Sidekiq::Job
+  #      sidekiq_options retry: 10, retry_queue: 'low'
   #    end
   #
   class JobRetry
     class Handled < ::RuntimeError; end
+
     class Skip < Handled; end
 
-    include Sidekiq::Util
+    include Sidekiq::Component
 
     DEFAULT_MAX_RETRY_ATTEMPTS = 25
 
-    def initialize(options = {})
-      @max_retries = Sidekiq.options.merge(options).fetch(:max_retries, DEFAULT_MAX_RETRY_ATTEMPTS)
+    def initialize(options)
+      @config = options
+      @max_retries = @config[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
     end
 
     # The global retry handler requires only the barest of data.
     # We want to be able to retry as much as possible so we don't
-    # require the worker to be instantiated.
-    def global(msg, queue)
+    # require the job to be instantiated.
+    def global(jobstr, queue)
       yield
     rescue Handled => ex
       raise ex
@@ -81,31 +87,29 @@ module Sidekiq
       # ignore, will be pushed back onto queue during hard_shutdown
       raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
 
-      if msg['retry']
-        attempt_retry(nil, msg, queue, e)
+      msg = Sidekiq.load_json(jobstr)
+      if msg["retry"]
+        process_retry(nil, msg, queue, e)
       else
         Sidekiq.death_handlers.each do |handler|
-          begin
-            handler.call(msg, e)
-          rescue => handler_ex
-            handle_exception(handler_ex, { context: "Error calling death handler", job: msg })
-          end
+          handler.call(msg, e)
+        rescue => handler_ex
+          handle_exception(handler_ex, {context: "Error calling death handler", job: msg})
         end
       end
 
       raise Handled
     end
 
-
     # The local retry support means that any errors that occur within
-    # this block can be associated with the given worker instance.
+    # this block can be associated with the given job instance.
     # This is required to support the `sidekiq_retries_exhausted` block.
     #
     # Note that any exception from the block is wrapped in the Skip
     # exception so the global block does not reprocess the error.  The
     # Skip exception is unwrapped within Sidekiq::Processor#process before
     # calling the handle_exception handlers.
-    def local(worker, msg, queue)
+    def local(jobinst, jobstr, queue)
       yield
     rescue Handled => ex
       raise ex
@@ -116,12 +120,13 @@ module Sidekiq
       # ignore, will be pushed back onto queue during hard_shutdown
       raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
 
-      if msg['retry'] == nil
-        msg['retry'] = worker.class.get_sidekiq_options['retry']
+      msg = Sidekiq.load_json(jobstr)
+      if msg["retry"].nil?
+        msg["retry"] = jobinst.class.get_sidekiq_options["retry"]
       end
 
-      raise e unless msg['retry']
-      attempt_retry(worker, msg, queue, e)
+      raise e unless msg["retry"]
+      process_retry(jobinst, msg, queue, e)
       # We've handled this error associated with this job, don't
       # need to handle it at the global level
       raise Skip
@@ -129,17 +134,13 @@ module Sidekiq
 
     private
 
-    # Note that +worker+ can be nil here if an error is raised before we can
-    # instantiate the worker instance.  All access must be guarded and
+    # Note that +jobinst+ can be nil here if an error is raised before we can
+    # instantiate the job instance.  All access must be guarded and
     # best effort.
-    def attempt_retry(worker, msg, queue, exception)
-      max_retry_attempts = retry_attempts_from(msg['retry'], @max_retries)
+    def process_retry(jobinst, msg, queue, exception)
+      max_retry_attempts = retry_attempts_from(msg["retry"], @max_retries)
 
-      msg['queue'] = if msg['retry_queue']
-        msg['retry_queue']
-      else
-        queue
-      end
+      msg["queue"] = (msg["retry_queue"] || queue)
 
       m = exception_message(exception)
       if m.respond_to?(:scrub!)
@@ -147,62 +148,100 @@ module Sidekiq
         m.scrub!
       end
 
-      msg['error_message'] = m
-      msg['error_class'] = exception.class.name
-      count = if msg['retry_count']
-        msg['retried_at'] = Time.now.to_f
-        msg['retry_count'] += 1
+      msg["error_message"] = m
+      msg["error_class"] = exception.class.name
+      count = if msg["retry_count"]
+        msg["retried_at"] = Time.now.to_f
+        msg["retry_count"] += 1
       else
-        msg['failed_at'] = Time.now.to_f
-        msg['retry_count'] = 0
+        msg["failed_at"] = Time.now.to_f
+        msg["retry_count"] = 0
       end
 
-      if msg['backtrace'] == true
-        msg['error_backtrace'] = exception.backtrace
-      elsif !msg['backtrace']
-        # do nothing
-      elsif msg['backtrace'].to_i != 0
-        msg['error_backtrace'] = exception.backtrace[0...msg['backtrace'].to_i]
+      if msg["backtrace"]
+        lines = if msg["backtrace"] == true
+          exception.backtrace
+        else
+          exception.backtrace[0...msg["backtrace"].to_i]
+        end
+
+        msg["error_backtrace"] = compress_backtrace(lines)
       end
 
-      if count < max_retry_attempts
-        delay = delay_for(worker, count, exception)
-        # Logging here can break retries if the logging device raises ENOSPC #3979
-        #logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
-        retry_at = Time.now.to_f + delay
-        payload = Sidekiq.dump_json(msg)
-        Sidekiq.redis do |conn|
-          conn.zadd('retry', retry_at.to_s, payload)
-        end
-      else
-        # Goodbye dear message, you (re)tried your best I'm sure.
-        retries_exhausted(worker, msg, exception)
+      # Goodbye dear message, you (re)tried your best I'm sure.
+      return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
+
+      strategy, delay = delay_for(jobinst, count, exception)
+      case strategy
+      when :discard
+        return # poof!
+      when :kill
+        return retries_exhausted(jobinst, msg, exception)
       end
+
+      # Logging here can break retries if the logging device raises ENOSPC #3979
+      # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
+      jitter = rand(10) * (count + 1)
+      retry_at = Time.now.to_f + delay + jitter
+      payload = Sidekiq.dump_json(msg)
+      redis do |conn|
+        conn.zadd("retry", retry_at.to_s, payload)
+      end
+    end
+
+    # returns (strategy, seconds)
+    def delay_for(jobinst, count, exception)
+      rv = begin
+        # sidekiq_retry_in can return two different things:
+        # 1. When to retry next, as an integer of seconds
+        # 2. A symbol which re-routes the job elsewhere, e.g. :discard, :kill, :default
+        jobinst&.sidekiq_retry_in_block&.call(count, exception)
+      rescue Exception => e
+        handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
+        nil
+      end
+
+      delay = (count**4) + 15
+      if Integer === rv && rv > 0
+        delay = rv
+      elsif rv == :discard
+        return [:discard, nil] # do nothing, job goes poof
+      elsif rv == :kill
+        return [:kill, nil]
+      end
+
+      [:default, delay]
     end
 
-    def retries_exhausted(worker, msg, exception)
+    def retries_exhausted(jobinst, msg, exception)
       begin
-        block = worker && worker.sidekiq_retries_exhausted_block
-        block.call(msg, exception) if block
+        block = jobinst&.sidekiq_retries_exhausted_block
+        block&.call(msg, exception)
       rescue => e
-        handle_exception(e, { context: "Error calling retries_exhausted", job: msg })
+        handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
       end
 
-      Sidekiq.death_handlers.each do |handler|
-        begin
-          handler.call(msg, exception)
-        rescue => e
-          handle_exception(e, { context: "Error calling death handler", job: msg })
-        end
-      end
+      send_to_morgue(msg) unless msg["dead"] == false
 
-      send_to_morgue(msg) unless msg['dead'] == false
+      config.death_handlers.each do |handler|
+        handler.call(msg, exception)
+      rescue => e
+        handle_exception(e, {context: "Error calling death handler", job: msg})
+      end
     end
 
     def send_to_morgue(msg)
-      logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
+      logger.info { "Adding dead #{msg["class"]} job #{msg["jid"]}" }
       payload = Sidekiq.dump_json(msg)
-      DeadSet.new.kill(payload, notify_failure: false)
+      now = Time.now.to_f
+
+      config.redis do |conn|
+        conn.multi do |xa|
+          xa.zadd("dead", now.to_s, payload)
+          xa.zremrangebyscore("dead", "-inf", now - config[:dead_timeout_in_seconds])
+          xa.zremrangebyrank("dead", 0, - config[:dead_max_jobs])
+        end
+      end
     end
 
     def retry_attempts_from(msg_retry, default)
@@ -213,28 +252,6 @@ module Sidekiq
       end
     end
 
-    def delay_for(worker, count, exception)
-      if worker && worker.sidekiq_retry_in_block
-        custom_retry_in = retry_in(worker, count, exception).to_i
-        return custom_retry_in if custom_retry_in > 0
-      end
-      seconds_to_delay(count)
-    end
-
-    # delayed_job uses the same basic formula
-    def seconds_to_delay(count)
-      (count ** 4) + 15 + (rand(30)*(count+1))
-    end
-
-    def retry_in(worker, count, exception)
-      begin
-        worker.sidekiq_retry_in_block.call(count, exception)
-      rescue Exception => e
-        handle_exception(e, { context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
-        nil
-      end
-    end
-
     def exception_caused_by_shutdown?(e, checked_causes = [])
       return false unless e.cause
 
@@ -249,14 +266,17 @@ module Sidekiq
     # Extract message from exception.
     # Set a default if the message raises an error
     def exception_message(exception)
-      begin
-        # App code can stuff all sorts of crazy binary data into the error message
-        # that won't convert to JSON.
-        exception.message.to_s[0, 10_000]
-      rescue
-        "!!! ERROR MESSAGE THREW AN ERROR !!!".dup
-      end
+      # App code can stuff all sorts of crazy binary data into the error message
+      # that won't convert to JSON.
+      exception.message.to_s[0, 10_000]
+    rescue
+      +"!!! ERROR MESSAGE THREW AN ERROR !!!"
     end
 
+    def compress_backtrace(backtrace)
+      serialized = Sidekiq.dump_json(backtrace)
+      compressed = Zlib::Deflate.deflate(serialized)
+      Base64.encode64(compressed)
+    end
   end
 end

data/lib/sidekiq/job_util.rb

@@ -0,0 +1,71 @@
+require "securerandom"
+require "time"
+
+module Sidekiq
+  module JobUtil
+    # These functions encapsulate various job utilities.
+
+    TRANSIENT_ATTRIBUTES = %w[]
+
+    def validate(item)
+      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
+      raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
+      raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
+      raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
+      raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
+    end
+
+    def verify_json(item)
+      job_class = item["wrapped"] || item["class"]
+      if Sidekiq[:on_complex_arguments] == :raise
+        msg = <<~EOM
+          Job arguments to #{job_class} must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices.
+          To disable this error, remove `Sidekiq.strict_args!` from your initializer.
+        EOM
+        raise(ArgumentError, msg) unless json_safe?(item)
+      elsif Sidekiq[:on_complex_arguments] == :warn
+        Sidekiq.logger.warn <<~EOM unless json_safe?(item)
+          Job arguments to #{job_class} do not serialize to JSON safely. This will raise an error in
+          Sidekiq 7.0. See https://github.com/mperham/sidekiq/wiki/Best-Practices or raise an error today
+          by calling `Sidekiq.strict_args!` during Sidekiq initialization.
+        EOM
+      end
+    end
+
+    def normalize_item(item)
+      validate(item)
+
+      # merge in the default sidekiq_options for the item's class and/or wrapped element
+      # this allows ActiveJobs to control sidekiq_options too.
+      defaults = normalized_hash(item["class"])
+      defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?(:get_sidekiq_options)
+      item = defaults.merge(item)
+
+      raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
+
+      # remove job attributes which aren't necessary to persist into Redis
+      TRANSIENT_ATTRIBUTES.each { |key| item.delete(key) }
+
+      item["jid"] ||= SecureRandom.hex(12)
+      item["class"] = item["class"].to_s
+      item["queue"] = item["queue"].to_s
+      item["created_at"] ||= Time.now.to_f
+      item
+    end
+
+    def normalized_hash(item_class)
+      if item_class.is_a?(Class)
+        raise(ArgumentError, "Message must include a Sidekiq::Job class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?(:get_sidekiq_options)
+        item_class.get_sidekiq_options
+      else
+        Sidekiq.default_job_options
+      end
+    end
+
+    private
+
+    def json_safe?(item)
+      JSON.parse(JSON.dump(item["args"])) == item["args"]
+    end
+  end
+end