sidekiq 6.5.10 → 8.1.7

data/lib/sidekiq/job.rb

@@ -1,13 +1,400 @@
-require "sidekiq/worker"
+# frozen_string_literal: true
+
+require "sidekiq/client"
 
 module Sidekiq
-  # Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
-  # Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
-  #
-  # The term "worker" is too generic and overly confusing, used in several
-  # different contexts meaning different things. Many people call a Sidekiq
-  # process a "worker". Some people call the thread that executes jobs a
-  # "worker". This change brings Sidekiq closer to ActiveJob where your job
-  # classes extend ApplicationJob.
-  Job = Worker
+  ##
+  # Include this module in your job class and you can easily create
+  # asynchronous jobs:
+  #
+  #   class HardJob
+  #     include Sidekiq::Job
+  #     sidekiq_options queue: 'critical', retry: 5
+  #
+  #     def perform(*args)
+  #       # do some work
+  #     end
+  #   end
+  #
+  # Then in your Rails app, you can do this:
+  #
+  #   HardJob.perform_async(1, 2, 3)
+  #
+  # Note that perform_async is a class method, perform is an instance method.
+  #
+  # Sidekiq::Job also includes several APIs to provide compatibility with
+  # ActiveJob.
+  #
+  #   class SomeJob
+  #     include Sidekiq::Job
+  #     queue_as :critical
+  #
+  #     def perform(...)
+  #     end
+  #   end
+  #
+  #   SomeJob.set(wait_until: 1.hour).perform_async(123)
+  #
+  # Note that arguments passed to the job must still obey Sidekiq's
+  # best practice for simple, JSON-native data types. Sidekiq will not
+  # implement ActiveJob's more complex argument serialization. For
+  # this reason, we don't implement `perform_later` as our call semantics
+  # are very different.
+  #
+  module Job
+    ##
+    # The Options module is extracted so we can include it in ActiveJob::Base
+    # and allow native AJs to configure Sidekiq features/internals.
+    module Options
+      def self.included(base)
+        base.extend(ClassMethods)
+        base.sidekiq_class_attribute :sidekiq_options_hash
+        base.sidekiq_class_attribute :sidekiq_retry_in_block
+        base.sidekiq_class_attribute :sidekiq_retries_exhausted_block
+      end
+
+      module ClassMethods
+        ACCESSOR_MUTEX = Mutex.new
+
+        ##
+        # Allows customization for this type of Job.
+        # Legal options:
+        #
+        #   queue - name of queue to use for this job type, default *default*
+        #   retry - enable retries for this Job in case of error during execution,
+        #      *true* to use the default or *Integer* count
+        #   backtrace - whether to save any error backtrace in the retry payload to display in web UI,
+        #      can be true, false or an integer number of lines to save, default *false*
+        #
+        # In practice, any option is allowed.  This is the main mechanism to configure the
+        # options for a specific job.
+        def sidekiq_options(opts = {})
+          # stringify 2 levels of keys
+          opts = opts.to_h do |k, v|
+            [k.to_s, (Hash === v) ? v.transform_keys(&:to_s) : v]
+          end
+
+          self.sidekiq_options_hash = get_sidekiq_options.merge(opts)
+        end
+
+        def sidekiq_retry_in(&block)
+          self.sidekiq_retry_in_block = block
+        end
+
+        def sidekiq_retries_exhausted(&block)
+          self.sidekiq_retries_exhausted_block = block
+        end
+
+        def get_sidekiq_options # :nodoc:
+          self.sidekiq_options_hash ||= Sidekiq.default_job_options
+        end
+
+        def sidekiq_class_attribute(*attrs)
+          instance_reader = true
+          instance_writer = true
+
+          attrs.each do |name|
+            synchronized_getter = "__synchronized_#{name}"
+
+            singleton_class.instance_eval do
+              undef_method(name) if method_defined?(name) || private_method_defined?(name)
+            end
+
+            define_singleton_method(synchronized_getter) { nil }
+            singleton_class.class_eval do
+              private(synchronized_getter)
+            end
+
+            define_singleton_method(name) { ACCESSOR_MUTEX.synchronize { send synchronized_getter } }
+
+            ivar = "@#{name}"
+
+            singleton_class.instance_eval do
+              m = "#{name}="
+              undef_method(m) if method_defined?(m) || private_method_defined?(m)
+            end
+            define_singleton_method(:"#{name}=") do |val|
+              singleton_class.class_eval do
+                ACCESSOR_MUTEX.synchronize do
+                  undef_method(synchronized_getter) if method_defined?(synchronized_getter) || private_method_defined?(synchronized_getter)
+                  define_method(synchronized_getter) { val }
+                end
+              end
+
+              if singleton_class?
+                class_eval do
+                  undef_method(name) if method_defined?(name) || private_method_defined?(name)
+                  define_method(name) do
+                    if instance_variable_defined? ivar
+                      instance_variable_get ivar
+                    else
+                      singleton_class.send name
+                    end
+                  end
+                end
+              end
+              val
+            end
+
+            if instance_reader
+              undef_method(name) if method_defined?(name) || private_method_defined?(name)
+              define_method(name) do
+                if instance_variable_defined?(ivar)
+                  instance_variable_get ivar
+                else
+                  self.class.public_send name
+                end
+              end
+            end
+
+            if instance_writer
+              m = "#{name}="
+              undef_method(m) if method_defined?(m) || private_method_defined?(m)
+              attr_writer name
+            end
+          end
+        end
+      end
+    end
+
+    attr_accessor :jid
+
+    # This attribute is implementation-specific and not a public API
+    attr_accessor :_context
+
+    def self.included(base)
+      raise ArgumentError, "Sidekiq::Job cannot be included in an ActiveJob: #{base.name}" if base.ancestors.any? { |c| c.name == "ActiveJob::Base" }
+
+      base.include(Options)
+      base.extend(ClassMethods)
+    end
+
+    def logger
+      Sidekiq.logger
+    end
+
+    def interrupted?
+      @_context&.stopping?
+    end
+
+    # This helper class encapsulates the set options for `set`, e.g.
+    #
+    #     SomeJob.set(queue: 'foo').perform_async(....)
+    #
+    class Setter
+      include Sidekiq::JobUtil
+
+      def initialize(klass, opts)
+        @klass = klass
+        # NB: the internal hash always has stringified keys
+        @opts = opts.transform_keys(&:to_s)
+
+        # ActiveJob compatibility
+        interval = @opts.delete("wait_until") || @opts.delete("wait")
+        at(interval) if interval
+      end
+
+      def set(options)
+        hash = options.transform_keys(&:to_s)
+        interval = hash.delete("wait_until") || hash.delete("wait")
+        @opts.merge!(hash)
+        at(interval) if interval
+        self
+      end
+
+      def perform_async(*args)
+        if @opts["sync"] == true
+          perform_inline(*args)
+        else
+          @klass.client_push(@opts.merge("args" => args, "class" => @klass))
+        end
+      end
+
+      # Explicit inline execution of a job. Returns nil if the job did not
+      # execute, true otherwise.
+      def perform_inline(*args)
+        raw = @opts.merge("args" => args, "class" => @klass)
+
+        # validate and normalize payload
+        item = normalize_item(raw)
+        queue = item["queue"]
+
+        # run client-side middleware
+        cfg = Sidekiq.default_configuration
+        result = cfg.client_middleware.invoke(item["class"], item, queue, cfg.redis_pool) do
+          item
+        end
+        return nil unless result
+
+        verify_json(item)
+
+        # round-trip the payload via JSON
+        msg = Sidekiq.load_json(Sidekiq.dump_json(item))
+
+        # prepare the job instance
+        klass = Object.const_get(msg["class"])
+        job = klass.new
+        job.jid = msg["jid"]
+        job.bid = msg["bid"] if job.respond_to?(:bid)
+
+        # run the job through server-side middleware
+        result = cfg.server_middleware.invoke(job, msg, msg["queue"]) do
+          # perform it
+          job.perform(*msg["args"])
+          true
+        end
+        return nil unless result
+        # jobs do not return a result. they should store any
+        # modified state.
+        true
+      end
+      alias_method :perform_sync, :perform_inline
+
+      def perform_bulk(args, **options)
+        client = @klass.build_client
+        client.push_bulk(@opts.merge({"class" => @klass, "args" => args}, options))
+      end
+
+      # +interval+ must be a timestamp, numeric or something that acts
+      #   numeric (like an activesupport time interval).
+      def perform_in(interval, *args)
+        at(interval).perform_async(*args)
+      end
+      alias_method :perform_at, :perform_in
+
+      private
+
+      def at(interval)
+        int = interval.to_f
+        now = Time.now.to_f
+        ts = ((int < 1_000_000_000) ? now + int : int)
+        # Optimization to enqueue something now that is scheduled to go out now or in the past
+        @opts["at"] = ts if ts > now
+        self
+      end
+    end
+
+    module ClassMethods
+      def delay(*args)
+        raise ArgumentError, "Do not call .delay on a Sidekiq::Job class, call .perform_async"
+      end
+
+      def delay_for(*args)
+        raise ArgumentError, "Do not call .delay_for on a Sidekiq::Job class, call .perform_in"
+      end
+
+      def delay_until(*args)
+        raise ArgumentError, "Do not call .delay_until on a Sidekiq::Job class, call .perform_at"
+      end
+
+      def queue_as(q)
+        sidekiq_options("queue" => q.to_s)
+      end
+
+      def set(options)
+        Setter.new(self, options)
+      end
+
+      def perform_async(*args)
+        Setter.new(self, {}).perform_async(*args)
+      end
+
+      # Inline execution of job's perform method after passing through Sidekiq.client_middleware and Sidekiq.server_middleware
+      def perform_inline(*args)
+        Setter.new(self, {}).perform_inline(*args)
+      end
+      alias_method :perform_sync, :perform_inline
+
+      ##
+      # Push a large number of jobs to Redis, while limiting the batch of
+      # each job payload to 1,000. This method helps cut down on the number
+      # of round trips to Redis, which can increase the performance of enqueueing
+      # large numbers of jobs.
+      #
+      # +items+ must be an Array of Arrays.
+      #
+      # The +:at+ option schedules the jobs for future execution. It accepts
+      # either a single Numeric timestamp (or seconds-from-now) applied to every
+      # job, or an Array of Numeric values with the same size as +items+ to
+      # schedule each job at its corresponding time.
+      #
+      # For finer-grained control, use `Sidekiq::Client.push_bulk` directly.
+      #
+      # Example (3 Redis round trips):
+      #
+      #     SomeJob.perform_async(1)
+      #     SomeJob.perform_async(2)
+      #     SomeJob.perform_async(3)
+      #
+      # Would instead become (1 Redis round trip):
+      #
+      #     SomeJob.perform_bulk([[1], [2], [3]])
+      #
+      # Scheduling every job 60 seconds from now (single Numeric +:at+):
+      #
+      #     SomeJob.perform_bulk([[1], [2], [3]], at: 60)
+      #
+      # Scheduling each job at its own time (Array +:at+):
+      #
+      #     SomeJob.perform_bulk([[1], [2]], at: [Time.now.to_f + 30, Time.now.to_f + 60])
+      #
+      def perform_bulk(*args, **kwargs)
+        Setter.new(self, {}).perform_bulk(*args, **kwargs)
+      end
+
+      # +interval+ must be a timestamp, numeric or something that acts
+      #   numeric (like an activesupport time interval).
+      def perform_in(interval, *args)
+        int = interval.to_f
+        now = Time.now.to_f
+        ts = ((int < 1_000_000_000) ? now + int : int)
+
+        item = {"class" => self, "args" => args}
+
+        # Optimization to enqueue something now that is scheduled to go out now or in the past
+        item["at"] = ts if ts > now
+
+        client_push(item)
+      end
+      alias_method :perform_at, :perform_in
+
+      ##
+      # Allows customization for this type of Job.
+      # Legal options:
+      #
+      #   queue - use a named queue for this Job, default 'default'
+      #   retry - enable the RetryJobs middleware for this Job, *true* to use the default
+      #      or *Integer* count
+      #   backtrace - whether to save any error backtrace in the retry payload to display in web UI,
+      #      can be true, false or an integer number of lines to save, default *false*
+      #   pool - use the given Redis connection pool to push this type of job to a given shard.
+      #
+      # In practice, any option is allowed.  This is the main mechanism to configure the
+      # options for a specific job.
+      def sidekiq_options(opts = {})
+        super
+      end
+
+      def client_push(item) # :nodoc:
+        raise ArgumentError, "Job payloads should contain no Symbols: #{item}" if item.any? { |k, v| k.is_a?(::Symbol) }
+
+        # allow the user to dynamically re-target jobs to another shard using the "pool" attribute
+        #   FooJob.set(pool: SOME_POOL).perform_async
+        old = Thread.current[:sidekiq_redis_pool]
+        pool = item.delete("pool")
+        Thread.current[:sidekiq_redis_pool] = pool if pool
+        begin
+          build_client.push(item)
+        ensure
+          Thread.current[:sidekiq_redis_pool] = old
+        end
+      end
+
+      def build_client # :nodoc:
+        pool = Thread.current[:sidekiq_redis_pool] || get_sidekiq_options["pool"] || Sidekiq.default_configuration.redis_pool
+        client_class = Thread.current[:sidekiq_client_class] || get_sidekiq_options["client_class"] || Sidekiq::Client
+        client_class.new(pool: pool)
+      end
+    end
+  end
 end

data/lib/sidekiq/job_logger.rb

@@ -2,22 +2,23 @@
 
 module Sidekiq
   class JobLogger
-    def initialize(logger = Sidekiq.logger)
-      @logger = logger
+    def initialize(config)
+      @config = config
+      @logger = @config.logger
+      @skip = !!@config[:skip_default_job_logging]
     end
 
     def call(item, queue)
       start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      @logger.info("start")
+      @logger.info { "start" } unless @skip
 
       yield
 
       Sidekiq::Context.add(:elapsed, elapsed(start))
-      @logger.info("done")
+      @logger.info { "done" } unless @skip
     rescue Exception
       Sidekiq::Context.add(:elapsed, elapsed(start))
-      @logger.info("fail")
-
+      @logger.info { "fail" } unless @skip
       raise
     end
 
@@ -25,16 +26,18 @@ module Sidekiq
       # If we're using a wrapper class, like ActiveJob, use the "wrapped"
       # attribute to expose the underlying thing.
       h = {
-        class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
-        jid: job_hash["jid"]
+        jid: job_hash["jid"],
+        class: job_hash["wrapped"] || job_hash["class"]
       }
-      h[:bid] = job_hash["bid"] if job_hash.has_key?("bid")
-      h[:tags] = job_hash["tags"] if job_hash.has_key?("tags")
+
+      @config[:logged_job_attributes].each do |attr|
+        h[attr.to_sym] = job_hash[attr] if job_hash.has_key?(attr)
+      end
 
       Thread.current[:sidekiq_context] = h
       level = job_hash["log_level"]
       if level
-        @logger.log_at(level, &block)
+        @logger.with_level(level, &block)
       else
         yield
       end

data/lib/sidekiq/job_retry.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "zlib"
-require "base64"
 require "sidekiq/component"
 
 module Sidekiq
@@ -49,7 +48,7 @@ module Sidekiq
   # The default number of retries is 25 which works out to about 3 weeks
   # You can change the default maximum number of retries in your initializer:
   #
-  #   Sidekiq.options[:max_retries] = 7
+  #   Sidekiq.default_configuration[:max_retries] = 7
   #
   # or limit the number of retries for a particular job and send retries to
   # a low priority queue with:
@@ -60,17 +59,23 @@ module Sidekiq
   #    end
   #
   class JobRetry
+    # Handled means the job failed but has been dealt with
+    # (by creating a retry, rescheduling it, etc). It still
+    # needs to be logged and dispatched to error_handlers.
     class Handled < ::RuntimeError; end
 
+    # Skip means the job failed but Sidekiq does not need to
+    # create a retry, log it or send to error_handlers.
     class Skip < Handled; end
 
     include Sidekiq::Component
 
     DEFAULT_MAX_RETRY_ATTEMPTS = 25
 
-    def initialize(options)
-      @config = options
-      @max_retries = @config[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
+    def initialize(cfg)
+      @config = cfg
+      @max_retries = Sidekiq.default_configuration[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
+      @backtrace_cleaner = Sidekiq.default_configuration[:backtrace_cleaner]
     end
 
     # The global retry handler requires only the barest of data.
@@ -91,7 +96,7 @@ module Sidekiq
       if msg["retry"]
         process_retry(nil, msg, queue, e)
       else
-        Sidekiq.death_handlers.each do |handler|
+        @config[:death_handlers].each do |handler|
           handler.call(msg, e)
         rescue => handler_ex
           handle_exception(handler_ex, {context: "Error calling death handler", job: msg})
@@ -129,11 +134,15 @@ module Sidekiq
       process_retry(jobinst, msg, queue, e)
       # We've handled this error associated with this job, don't
       # need to handle it at the global level
-      raise Skip
+      raise Handled
     end
 
     private
 
+    def now_ms
+      ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
+    end
+
     # Note that +jobinst+ can be nil here if an error is raised before we can
     # instantiate the job instance.  All access must be guarded and
     # best effort.
@@ -144,44 +153,53 @@ module Sidekiq
 
       m = exception_message(exception)
       if m.respond_to?(:scrub!)
-        m.force_encoding("utf-8")
+        m.force_encoding(Encoding::UTF_8)
         m.scrub!
       end
 
       msg["error_message"] = m
       msg["error_class"] = exception.class.name
       count = if msg["retry_count"]
-        msg["retried_at"] = Time.now.to_f
+        msg["retried_at"] = now_ms
         msg["retry_count"] += 1
       else
-        msg["failed_at"] = Time.now.to_f
+        msg["failed_at"] = now_ms
         msg["retry_count"] = 0
       end
 
       if msg["backtrace"]
+        backtrace = @backtrace_cleaner.call(exception.backtrace)
         lines = if msg["backtrace"] == true
-          exception.backtrace
+          backtrace
         else
-          exception.backtrace[0...msg["backtrace"].to_i]
+          backtrace[0...msg["backtrace"].to_i]
         end
 
         msg["error_backtrace"] = compress_backtrace(lines)
       end
 
-      # Goodbye dear message, you (re)tried your best I'm sure.
-      return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
+      # retry_for and retry are mutually exclusive - if retry_for is set,
+      # we exclusively use duration-based retry logic and ignore count-based logic
+      rf = msg["retry_for"]
+      if rf
+        return retries_exhausted(jobinst, msg, exception) if (time_for(msg["failed_at"]) + rf) < Time.now
+      elsif count >= max_retry_attempts
+        return retries_exhausted(jobinst, msg, exception)
+      end
 
-      strategy, delay = delay_for(jobinst, count, exception)
+      strategy, delay = delay_for(jobinst, count, exception, msg)
       case strategy
       when :discard
-        return # poof!
+        msg["discarded_at"] = now_ms
+
+        return run_death_handlers(msg, exception)
       when :kill
         return retries_exhausted(jobinst, msg, exception)
       end
 
       # Logging here can break retries if the logging device raises ENOSPC #3979
       # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
-      jitter = rand(10) * (count + 1)
+      jitter = rand(10 * (count + 1))
       retry_at = Time.now.to_f + delay + jitter
       payload = Sidekiq.dump_json(msg)
       redis do |conn|
@@ -189,18 +207,34 @@ module Sidekiq
       end
     end
 
+    def time_for(item)
+      if item.is_a?(Float)
+        Time.at(item)
+      else
+        Time.at(item / 1000, item % 1000, :millisecond)
+      end
+    end
+
     # returns (strategy, seconds)
-    def delay_for(jobinst, count, exception)
+    def delay_for(jobinst, count, exception, msg)
       rv = begin
         # sidekiq_retry_in can return two different things:
         # 1. When to retry next, as an integer of seconds
         # 2. A symbol which re-routes the job elsewhere, e.g. :discard, :kill, :default
-        jobinst&.sidekiq_retry_in_block&.call(count, exception)
+        block = jobinst&.sidekiq_retry_in_block
+
+        # the sidekiq_retry_in_block can be defined in a wrapped class (ActiveJob for instance)
+        unless msg["wrapped"].nil?
+          wrapped = Object.const_get(msg["wrapped"])
+          block = wrapped.respond_to?(:sidekiq_retry_in_block) ? wrapped.sidekiq_retry_in_block : nil
+        end
+        block&.call(count, exception, msg)
       rescue Exception => e
         handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
         nil
       end
 
+      rv = rv.to_i if rv.respond_to?(:to_i)
       delay = (count**4) + 15
       if Integer === rv && rv > 0
         delay = rv
@@ -214,19 +248,35 @@ module Sidekiq
     end
 
     def retries_exhausted(jobinst, msg, exception)
-      begin
+      rv = begin
         block = jobinst&.sidekiq_retries_exhausted_block
+
+        # the sidekiq_retries_exhausted_block can be defined in a wrapped class (ActiveJob for instance)
+        unless msg["wrapped"].nil?
+          wrapped = Object.const_get(msg["wrapped"])
+          block = wrapped.respond_to?(:sidekiq_retries_exhausted_block) ? wrapped.sidekiq_retries_exhausted_block : nil
+        end
         block&.call(msg, exception)
       rescue => e
         handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
       end
 
-      send_to_morgue(msg) unless msg["dead"] == false
+      discarded = msg["dead"] == false || rv == :discard
+
+      if discarded
+        msg["discarded_at"] = now_ms
+      else
+        send_to_morgue(msg)
+      end
+
+      run_death_handlers(msg, exception)
+    end
 
-      config.death_handlers.each do |handler|
-        handler.call(msg, exception)
+    def run_death_handlers(job, exception)
+      @config[:death_handlers].each do |handler|
+        handler.call(job, exception)
       rescue => e
-        handle_exception(e, {context: "Error calling death handler", job: msg})
+        handle_exception(e, {context: "Error calling death handler", job: job})
       end
     end
 
@@ -235,11 +285,11 @@ module Sidekiq
       payload = Sidekiq.dump_json(msg)
       now = Time.now.to_f
 
-      config.redis do |conn|
+      redis do |conn|
         conn.multi do |xa|
           xa.zadd("dead", now.to_s, payload)
-          xa.zremrangebyscore("dead", "-inf", now - config[:dead_timeout_in_seconds])
-          xa.zremrangebyrank("dead", 0, - config[:dead_max_jobs])
+          xa.zremrangebyscore("dead", "-inf", now - @config[:dead_timeout_in_seconds])
+          xa.zremrangebyrank("dead", 0, - @config[:dead_max_jobs])
         end
       end
     end
@@ -276,7 +326,7 @@ module Sidekiq
     def compress_backtrace(backtrace)
       serialized = Sidekiq.dump_json(backtrace)
       compressed = Zlib::Deflate.deflate(serialized)
-      Base64.encode64(compressed)
+      [compressed].pack("m0") # Base64.strict_encode64
     end
   end
 end