sidekiq 7.0.0 → 7.3.0

data/lib/sidekiq/job/iterable.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+require_relative "iterable/enumerators"
+
+module Sidekiq
+  module Job
+    class Interrupted < ::RuntimeError; end
+
+    module Iterable
+      include Enumerators
+
+      # @api private
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      # @api private
+      module ClassMethods
+        def method_added(method_name)
+          raise "#{self} is an iterable job and must not define #perform" if method_name == :perform
+          super
+        end
+      end
+
+      # @api private
+      def initialize
+        super
+
+        @_executions = 0
+        @_cursor = nil
+        @_start_time = nil
+        @_runtime = 0
+      end
+
+      # A hook to override that will be called when the job starts iterating.
+      #
+      # It is called only once, for the first time.
+      #
+      def on_start
+      end
+
+      # A hook to override that will be called around each iteration.
+      #
+      # Can be useful for some metrics collection, performance tracking etc.
+      #
+      def around_iteration
+        yield
+      end
+
+      # A hook to override that will be called when the job resumes iterating.
+      #
+      def on_resume
+      end
+
+      # A hook to override that will be called each time the job is interrupted.
+      #
+      # This can be due to interruption or sidekiq stopping.
+      #
+      def on_stop
+      end
+
+      # A hook to override that will be called when the job finished iterating.
+      #
+      def on_complete
+      end
+
+      # The enumerator to be iterated over.
+      #
+      # @return [Enumerator]
+      #
+      # @raise [NotImplementedError] with a message advising subclasses to
+      #     implement an override for this method.
+      #
+      def build_enumerator(*)
+        raise NotImplementedError, "#{self.class.name} must implement a '#build_enumerator' method"
+      end
+
+      # The action to be performed on each item from the enumerator.
+      #
+      # @return [void]
+      #
+      # @raise [NotImplementedError] with a message advising subclasses to
+      #     implement an override for this method.
+      #
+      def each_iteration(*)
+        raise NotImplementedError, "#{self.class.name} must implement an '#each_iteration' method"
+      end
+
+      def iteration_key
+        "it-#{jid}"
+      end
+
+      # @api private
+      def perform(*arguments)
+        fetch_previous_iteration_state
+
+        @_executions += 1
+        @_start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+        enumerator = build_enumerator(*arguments, cursor: @_cursor)
+        unless enumerator
+          logger.info("'#build_enumerator' returned nil, skipping the job.")
+          return
+        end
+
+        assert_enumerator!(enumerator)
+
+        if @_executions == 1
+          on_start
+        else
+          on_resume
+        end
+
+        completed = catch(:abort) do
+          iterate_with_enumerator(enumerator, arguments)
+        end
+
+        on_stop
+        completed = handle_completed(completed)
+
+        if completed
+          on_complete
+          cleanup
+        else
+          reenqueue_iteration_job
+        end
+      end
+
+      private
+
+      def fetch_previous_iteration_state
+        state = Sidekiq.redis { |conn| conn.hgetall(iteration_key) }
+
+        unless state.empty?
+          @_executions = state["ex"].to_i
+          @_cursor = Sidekiq.load_json(state["c"])
+          @_runtime = state["rt"].to_f
+        end
+      end
+
+      STATE_FLUSH_INTERVAL = 5 # seconds
+      # we need to keep the state around as long as the job
+      # might be retrying
+      STATE_TTL = 30 * 24 * 60 * 60 # one month
+
+      def iterate_with_enumerator(enumerator, arguments)
+        found_record = false
+        state_flushed_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+        enumerator.each do |object, cursor|
+          found_record = true
+          @_cursor = cursor
+
+          is_interrupted = interrupted?
+          if ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - state_flushed_at >= STATE_FLUSH_INTERVAL || is_interrupted
+            flush_state
+            state_flushed_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+          end
+
+          return false if is_interrupted
+
+          around_iteration do
+            each_iteration(object, *arguments)
+          end
+        end
+
+        logger.debug("Enumerator found nothing to iterate!") unless found_record
+        true
+      ensure
+        @_runtime += (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @_start_time)
+      end
+
+      def reenqueue_iteration_job
+        flush_state
+        logger.debug { "Interrupting job (cursor=#{@_cursor.inspect})" }
+
+        raise Interrupted
+      end
+
+      def assert_enumerator!(enum)
+        unless enum.is_a?(Enumerator)
+          raise ArgumentError, <<~MSG
+            #build_enumerator must return an Enumerator, but returned #{enum.class}.
+            Example:
+              def build_enumerator(params, cursor:)
+                active_record_records_enumerator(
+                  Shop.find(params["shop_id"]).products,
+                  cursor: cursor
+                )
+              end
+          MSG
+        end
+      end
+
+      def flush_state
+        key = iteration_key
+        state = {
+          "ex" => @_executions,
+          "c" => Sidekiq.dump_json(@_cursor),
+          "rt" => @_runtime
+        }
+
+        Sidekiq.redis do |conn|
+          conn.multi do |pipe|
+            pipe.hset(key, state)
+            pipe.expire(key, STATE_TTL)
+          end
+        end
+      end
+
+      def cleanup
+        logger.debug {
+          format("Completed iteration. executions=%d runtime=%.3f", @_executions, @_runtime)
+        }
+        Sidekiq.redis { |conn| conn.unlink(iteration_key) }
+      end
+
+      def handle_completed(completed)
+        case completed
+        when nil, # someone aborted the job but wants to call the on_complete callback
+             true
+          true
+        when false
+          false
+        else
+          raise "Unexpected thrown value: #{completed.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/job.rb

@@ -69,7 +69,11 @@ module Sidekiq
         # In practice, any option is allowed.  This is the main mechanism to configure the
         # options for a specific job.
         def sidekiq_options(opts = {})
-          opts = opts.transform_keys(&:to_s) # stringify
+          # stringify 2 levels of keys
+          opts = opts.to_h do |k, v|
+            [k.to_s, (Hash === v) ? v.transform_keys(&:to_s) : v]
+          end
+
           self.sidekiq_options_hash = get_sidekiq_options.merge(opts)
         end
 
@@ -109,7 +113,7 @@ module Sidekiq
               m = "#{name}="
               undef_method(m) if method_defined?(m) || private_method_defined?(m)
             end
-            define_singleton_method("#{name}=") do |val|
+            define_singleton_method(:"#{name}=") do |val|
               singleton_class.class_eval do
                 ACCESSOR_MUTEX.synchronize do
                   undef_method(synchronized_getter) if method_defined?(synchronized_getter) || private_method_defined?(synchronized_getter)
@@ -155,6 +159,9 @@ module Sidekiq
 
     attr_accessor :jid
 
+    # This attribute is implementation-specific and not a public API
+    attr_accessor :_context
+
     def self.included(base)
       raise ArgumentError, "Sidekiq::Job cannot be included in an ActiveJob: #{base.name}" if base.ancestors.any? { |c| c.name == "ActiveJob::Base" }
 
@@ -166,6 +173,10 @@ module Sidekiq
       Sidekiq.logger
     end
 
+    def interrupted?
+      @_context&.stopping?
+    end
+
     # This helper class encapsulates the set options for `set`, e.g.
     #
     #     SomeJob.set(queue: 'foo').perform_async(....)
@@ -239,11 +250,7 @@ module Sidekiq
 
       def perform_bulk(args, batch_size: 1_000)
         client = @klass.build_client
-        result = args.each_slice(batch_size).flat_map do |slice|
-          client.push_bulk(@opts.merge("class" => @klass, "args" => slice))
-        end
-
-        result.is_a?(Enumerator::Lazy) ? result.force : result
+        client.push_bulk(@opts.merge("class" => @klass, "args" => args, :batch_size => batch_size))
       end
 
       # +interval+ must be a timestamp, numeric or something that acts
@@ -258,7 +265,7 @@ module Sidekiq
       def at(interval)
         int = interval.to_f
         now = Time.now.to_f
-        ts = (int < 1_000_000_000 ? now + int : int)
+        ts = ((int < 1_000_000_000) ? now + int : int)
         # Optimization to enqueue something now that is scheduled to go out now or in the past
         @opts["at"] = ts if ts > now
         self
@@ -325,7 +332,7 @@ module Sidekiq
       def perform_in(interval, *args)
         int = interval.to_f
         now = Time.now.to_f
-        ts = (int < 1_000_000_000 ? now + int : int)
+        ts = ((int < 1_000_000_000) ? now + int : int)
 
         item = {"class" => self, "args" => args}
 
@@ -370,7 +377,7 @@ module Sidekiq
 
       def build_client # :nodoc:
         pool = Thread.current[:sidekiq_redis_pool] || get_sidekiq_options["pool"] || Sidekiq.default_configuration.redis_pool
-        client_class = get_sidekiq_options["client_class"] || Sidekiq::Client
+        client_class = Thread.current[:sidekiq_client_class] || get_sidekiq_options["client_class"] || Sidekiq::Client
         client_class.new(pool: pool)
       end
     end

data/lib/sidekiq/job_logger.rb

@@ -2,23 +2,36 @@
 
 module Sidekiq
   class JobLogger
-    def initialize(logger)
+    include Sidekiq::Component
+
+    def initialize(config)
+      @config = config
       @logger = logger
     end
 
+    # If true we won't do any job logging out of the box.
+    # The user is responsible for any logging.
+    def skip_default_logging?
+      config[:skip_default_job_logging]
+    end
+
     def call(item, queue)
-      start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      @logger.info("start")
+      return yield if skip_default_logging?
 
-      yield
+      begin
+        start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        @logger.info("start")
 
-      Sidekiq::Context.add(:elapsed, elapsed(start))
-      @logger.info("done")
-    rescue Exception
-      Sidekiq::Context.add(:elapsed, elapsed(start))
-      @logger.info("fail")
+        yield
+
+        Sidekiq::Context.add(:elapsed, elapsed(start))
+        @logger.info("done")
+      rescue Exception
+        Sidekiq::Context.add(:elapsed, elapsed(start))
+        @logger.info("fail")
 
-      raise
+        raise
+      end
     end
 
     def prepare(job_hash, &block)
@@ -33,7 +46,7 @@ module Sidekiq
 
       Thread.current[:sidekiq_context] = h
       level = job_hash["log_level"]
-      if level
+      if level && @logger.respond_to?(:log_at)
         @logger.log_at(level, &block)
       else
         yield

data/lib/sidekiq/job_retry.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "zlib"
-require "base64"
 require "sidekiq/component"
 
 module Sidekiq
@@ -49,7 +48,7 @@ module Sidekiq
   # The default number of retries is 25 which works out to about 3 weeks
   # You can change the default maximum number of retries in your initializer:
   #
-  #   Sidekiq.options[:max_retries] = 7
+  #   Sidekiq.default_configuration[:max_retries] = 7
   #
   # or limit the number of retries for a particular job and send retries to
   # a low priority queue with:
@@ -60,8 +59,13 @@ module Sidekiq
   #    end
   #
   class JobRetry
+    # Handled means the job failed but has been dealt with
+    # (by creating a retry, rescheduling it, etc). It still
+    # needs to be logged and dispatched to error_handlers.
     class Handled < ::RuntimeError; end
 
+    # Skip means the job failed but Sidekiq does not need to
+    # create a retry, log it or send to error_handlers.
     class Skip < Handled; end
 
     include Sidekiq::Component
@@ -71,6 +75,7 @@ module Sidekiq
     def initialize(capsule)
       @config = @capsule = capsule
       @max_retries = Sidekiq.default_configuration[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
+      @backtrace_cleaner = Sidekiq.default_configuration[:backtrace_cleaner]
     end
 
     # The global retry handler requires only the barest of data.
@@ -129,7 +134,7 @@ module Sidekiq
       process_retry(jobinst, msg, queue, e)
       # We've handled this error associated with this job, don't
       # need to handle it at the global level
-      raise Skip
+      raise Handled
     end
 
     private
@@ -159,19 +164,22 @@ module Sidekiq
       end
 
       if msg["backtrace"]
+        backtrace = @backtrace_cleaner.call(exception.backtrace)
         lines = if msg["backtrace"] == true
-          exception.backtrace
+          backtrace
         else
-          exception.backtrace[0...msg["backtrace"].to_i]
+          backtrace[0...msg["backtrace"].to_i]
         end
 
         msg["error_backtrace"] = compress_backtrace(lines)
       end
 
-      # Goodbye dear message, you (re)tried your best I'm sure.
       return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
 
-      strategy, delay = delay_for(jobinst, count, exception)
+      rf = msg["retry_for"]
+      return retries_exhausted(jobinst, msg, exception) if rf && ((msg["failed_at"] + rf) < Time.now.to_f)
+
+      strategy, delay = delay_for(jobinst, count, exception, msg)
       case strategy
       when :discard
         return # poof!
@@ -190,17 +198,25 @@ module Sidekiq
     end
 
     # returns (strategy, seconds)
-    def delay_for(jobinst, count, exception)
+    def delay_for(jobinst, count, exception, msg)
       rv = begin
         # sidekiq_retry_in can return two different things:
         # 1. When to retry next, as an integer of seconds
         # 2. A symbol which re-routes the job elsewhere, e.g. :discard, :kill, :default
-        jobinst&.sidekiq_retry_in_block&.call(count, exception)
+        block = jobinst&.sidekiq_retry_in_block
+
+        # the sidekiq_retry_in_block can be defined in a wrapped class (ActiveJob for instance)
+        unless msg["wrapped"].nil?
+          wrapped = Object.const_get(msg["wrapped"])
+          block = wrapped.respond_to?(:sidekiq_retry_in_block) ? wrapped.sidekiq_retry_in_block : nil
+        end
+        block&.call(count, exception, msg)
       rescue Exception => e
         handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
         nil
       end
 
+      rv = rv.to_i if rv.respond_to?(:to_i)
       delay = (count**4) + 15
       if Integer === rv && rv > 0
         delay = rv
@@ -214,13 +230,20 @@ module Sidekiq
     end
 
     def retries_exhausted(jobinst, msg, exception)
-      begin
+      rv = begin
         block = jobinst&.sidekiq_retries_exhausted_block
+
+        # the sidekiq_retries_exhausted_block can be defined in a wrapped class (ActiveJob for instance)
+        unless msg["wrapped"].nil?
+          wrapped = Object.const_get(msg["wrapped"])
+          block = wrapped.respond_to?(:sidekiq_retries_exhausted_block) ? wrapped.sidekiq_retries_exhausted_block : nil
+        end
         block&.call(msg, exception)
       rescue => e
         handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
       end
 
+      return if rv == :discard # poof!
       send_to_morgue(msg) unless msg["dead"] == false
 
       @capsule.config.death_handlers.each do |handler|
@@ -276,7 +299,7 @@ module Sidekiq
     def compress_backtrace(backtrace)
       serialized = Sidekiq.dump_json(backtrace)
       compressed = Zlib::Deflate.deflate(serialized)
-      Base64.encode64(compressed)
+      [compressed].pack("m0") # Base64.strict_encode64
     end
   end
 end

data/lib/sidekiq/job_util.rb

@@ -9,26 +9,32 @@ module Sidekiq
 
     def validate(item)
       raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
-      raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
+      raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array) || item["args"].is_a?(Enumerator::Lazy)
       raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
       raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
       raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
+      raise(ArgumentError, "retry_for must be a relative amount of time, e.g. 48.hours `#{item}`") if item["retry_for"] && item["retry_for"] > 1_000_000_000
     end
 
     def verify_json(item)
       job_class = item["wrapped"] || item["class"]
-      if Sidekiq::Config::DEFAULTS[:on_complex_arguments] == :raise
-        msg = <<~EOM
-          Job arguments to #{job_class} must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices.
-          To disable this error, add `Sidekiq.strict_args!(false)` to your initializer.
-        EOM
-        raise(ArgumentError, msg) unless json_safe?(item)
-      elsif Sidekiq::Config::DEFAULTS[:on_complex_arguments] == :warn
-        warn <<~EOM unless json_safe?(item)
-          Job arguments to #{job_class} do not serialize to JSON safely. This will raise an error in
-          Sidekiq 7.0. See https://github.com/mperham/sidekiq/wiki/Best-Practices or raise an error today
-          by calling `Sidekiq.strict_args!` during Sidekiq initialization.
-        EOM
+      args = item["args"]
+      mode = Sidekiq::Config::DEFAULTS[:on_complex_arguments]
+
+      if mode == :raise || mode == :warn
+        if (unsafe_item = json_unsafe?(args))
+          msg = <<~EOM
+            Job arguments to #{job_class} must be native JSON types, but #{unsafe_item.inspect} is a #{unsafe_item.class}.
+            See https://github.com/sidekiq/sidekiq/wiki/Best-Practices
+            To disable this error, add `Sidekiq.strict_args!(false)` to your initializer.
+          EOM
+
+          if mode == :raise
+            raise(ArgumentError, msg)
+          else
+            warn(msg)
+          end
+        end
       end
     end
 
@@ -49,6 +55,7 @@ module Sidekiq
       item["jid"] ||= SecureRandom.hex(12)
       item["class"] = item["class"].to_s
       item["queue"] = item["queue"].to_s
+      item["retry_for"] = item["retry_for"].to_i if item["retry_for"]
       item["created_at"] ||= Time.now.to_f
       item
     end
@@ -64,8 +71,37 @@ module Sidekiq
 
     private
 
-    def json_safe?(item)
-      JSON.parse(JSON.dump(item["args"])) == item["args"]
+    RECURSIVE_JSON_UNSAFE = {
+      Integer => ->(val) {},
+      Float => ->(val) {},
+      TrueClass => ->(val) {},
+      FalseClass => ->(val) {},
+      NilClass => ->(val) {},
+      String => ->(val) {},
+      Array => ->(val) {
+        val.each do |e|
+          unsafe_item = RECURSIVE_JSON_UNSAFE[e.class].call(e)
+          return unsafe_item unless unsafe_item.nil?
+        end
+        nil
+      },
+      Hash => ->(val) {
+        val.each do |k, v|
+          return k unless String === k
+
+          unsafe_item = RECURSIVE_JSON_UNSAFE[v.class].call(v)
+          return unsafe_item unless unsafe_item.nil?
+        end
+        nil
+      }
+    }
+
+    RECURSIVE_JSON_UNSAFE.default = ->(val) { val }
+    RECURSIVE_JSON_UNSAFE.compare_by_identity
+    private_constant :RECURSIVE_JSON_UNSAFE
+
+    def json_unsafe?(item)
+      RECURSIVE_JSON_UNSAFE[item.class].call(item)
     end
   end
 end