sidekiq 7.0.3 → 7.1.2

data/lib/sidekiq/api.rb

@@ -92,11 +92,11 @@ module Sidekiq
           pipeline.zcard("retry")
           pipeline.zcard("dead")
           pipeline.scard("processes")
-          pipeline.lrange("queue:default", -1, -1)
+          pipeline.lindex("queue:default", -1)
         end
       }
 
-      default_queue_latency = if (entry = pipe1_res[6].first)
+      default_queue_latency = if (entry = pipe1_res[6])
         job = begin
           Sidekiq.load_json(entry)
         rescue
@@ -264,8 +264,8 @@ module Sidekiq
     # @return [Float] in seconds
     def latency
       entry = Sidekiq.redis { |conn|
-        conn.lrange(@rname, -1, -1)
-      }.first
+        conn.lindex(@rname, -1)
+      }
       return 0 unless entry
       job = Sidekiq.load_json(entry)
       now = Time.now.to_f
@@ -391,13 +391,13 @@ module Sidekiq
     def display_args
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
       @display_args ||= if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
-        job_args = self["wrapped"] ? args[0]["arguments"] : []
+        job_args = self["wrapped"] ? deserialize_argument(args[0]["arguments"]) : []
         if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
           # remove MailerClass, mailer_method and 'deliver_now'
           job_args.drop(3)
         elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
           # remove MailerClass, mailer_method and 'deliver_now'
-          job_args.drop(3).first["args"]
+          job_args.drop(3).first.values_at("params", "args")
         else
           job_args
         end
@@ -467,6 +467,29 @@ module Sidekiq
 
     private
 
+    ACTIVE_JOB_PREFIX = "_aj_"
+    GLOBALID_KEY = "_aj_globalid"
+
+    def deserialize_argument(argument)
+      case argument
+      when Array
+        argument.map { |arg| deserialize_argument(arg) }
+      when Hash
+        if serialized_global_id?(argument)
+          argument[GLOBALID_KEY]
+        else
+          argument.transform_values { |v| deserialize_argument(v) }
+            .reject { |k, _| k.start_with?(ACTIVE_JOB_PREFIX) }
+        end
+      else
+        argument
+      end
+    end
+
+    def serialized_global_id?(hash)
+      hash.size == 1 && hash.include?(GLOBALID_KEY)
+    end
+
     def uncompress_backtrace(backtrace)
       decoded = Base64.decode64(backtrace)
       uncompressed = Zlib::Inflate.inflate(decoded)
@@ -548,7 +571,7 @@ module Sidekiq
     def remove_job
       Sidekiq.redis do |conn|
         results = conn.multi { |transaction|
-          transaction.zrangebyscore(parent.name, score, score)
+          transaction.zrange(parent.name, score, score, "BYSCORE")
           transaction.zremrangebyscore(parent.name, score, score)
         }.first
 
@@ -683,7 +706,7 @@ module Sidekiq
         end
 
       elements = Sidekiq.redis { |conn|
-        conn.zrangebyscore(name, begin_score, end_score, withscores: true)
+        conn.zrange(name, begin_score, end_score, "BYSCORE", withscores: true)
       }
 
       elements.each_with_object([]) do |element, result|
@@ -702,7 +725,7 @@ module Sidekiq
     def find_job(jid)
       Sidekiq.redis do |conn|
         conn.zscan(name, match: "*#{jid}*", count: 100) do |entry, score|
-          job = JSON.parse(entry)
+          job = Sidekiq.load_json(entry)
           matched = job["jid"] == jid
           return SortedEntry.new(self, score, entry) if matched
         end
@@ -724,7 +747,7 @@ module Sidekiq
     # @api private
     def delete_by_jid(score, jid)
       Sidekiq.redis do |conn|
-        elements = conn.zrangebyscore(name, score, score)
+        elements = conn.zrange(name, score, score, "BYSCORE")
         elements.each do |element|
           if element.index(jid)
             message = Sidekiq.load_json(element)

data/lib/sidekiq/capsule.rb

@@ -28,6 +28,7 @@ module Sidekiq
       @name = name
       @config = config
       @queues = ["default"]
+      @weights = {"default" => 0}
       @concurrency = config[:concurrency]
       @mode = :strict
     end

data/lib/sidekiq/cli.rb

@@ -84,7 +84,7 @@ module Sidekiq # :nodoc:
 
           WARNING: Your Redis instance will evict Sidekiq data under heavy load.
           The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
-          See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
+          See: https://github.com/sidekiq/sidekiq/wiki/Using-Redis#memory
 
         EOM
       end
@@ -230,6 +230,7 @@ module Sidekiq # :nodoc:
       # Both Sinatra 2.0+ and Sidekiq support this term.
       # RAILS_ENV and RACK_ENV are there for legacy support.
       @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      config[:environment] = @environment
     end
 
     def symbolize_keys_deep!(hash)
@@ -396,7 +397,7 @@ module Sidekiq # :nodoc:
     end
 
     def parse_config(path)
-      erb = ERB.new(File.read(path))
+      erb = ERB.new(File.read(path), trim_mode: "-")
       erb.filename = File.expand_path(path)
       opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
 

data/lib/sidekiq/client.rb

@@ -96,8 +96,9 @@ module Sidekiq
 
     ##
     # Push a large number of jobs to Redis. This method cuts out the redis
-    # network round trip latency.  I wouldn't recommend pushing more than
-    # 1000 per call but YMMV based on network quality, size of job args, etc.
+    # network round trip latency. It pushes jobs in batches if more than
+    # `:batch_size` (1000 by default) of jobs are passed. I wouldn't recommend making `:batch_size`
+    # larger than 1000 but YMMV based on network quality, size of job args, etc.
     # A large number of jobs can cause a bit of Redis command processing latency.
     #
     # Takes the same arguments as #push except that args is expected to be
@@ -105,13 +106,15 @@ module Sidekiq
     # is run through the client middleware pipeline and each job gets its own Job ID
     # as normal.
     #
-    # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
-    # than the number given if the middleware stopped processing for one or more jobs.
+    # Returns an array of the of pushed jobs' jids, may contain nils if any client middleware
+    # prevented a job push.
+    #
+    # Example (pushing jobs in batches):
+    #   push_bulk('class' => 'MyJob', 'args' => (1..100_000).to_a, batch_size: 1_000)
+    #
     def push_bulk(items)
+      batch_size = items.delete(:batch_size) || items.delete("batch_size") || 1_000
       args = items["args"]
-      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
-      return [] if args.empty? # no jobs to push
-
       at = items.delete("at")
       raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
       raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
@@ -120,18 +123,26 @@ module Sidekiq
       raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
 
       normed = normalize_item(items)
-      payloads = args.map.with_index { |job_args, index|
-        copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
-        copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
-        result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
-          verify_json(copy)
-          copy
-        end
-        result || nil
-      }.compact
+      result = args.each_slice(batch_size).flat_map do |slice|
+        raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless slice.is_a?(Array) && slice.all?(Array)
+        break [] if slice.empty? # no jobs to push
+
+        payloads = slice.map.with_index { |job_args, index|
+          copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
+          copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
+          result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
+            verify_json(copy)
+            copy
+          end
+          result || nil
+        }
+
+        to_push = payloads.compact
+        raw_push(to_push) unless to_push.empty?
+        payloads.map { |payload| payload&.[]("jid") }
+      end
 
-      raw_push(payloads) unless payloads.empty?
-      payloads.collect { |payload| payload["jid"] }
+      result.is_a?(Enumerator::Lazy) ? result.force : result
     end
 
     # Allows sharding of jobs across any number of Redis instances.  All jobs
@@ -160,8 +171,8 @@ module Sidekiq
         new.push(item)
       end
 
-      def push_bulk(items)
-        new.push_bulk(items)
+      def push_bulk(...)
+        new.push_bulk(...)
       end
 
       # Resque compatibility helpers.  Note all helpers
@@ -235,6 +246,8 @@ module Sidekiq
       if payloads.first.key?("at")
         conn.zadd("schedule", payloads.flat_map { |hash|
           at = hash.delete("at").to_s
+          # ActiveJob sets this but the job has not been enqueued yet
+          hash.delete("enqueued_at")
           [at, Sidekiq.dump_json(hash)]
         })
       else

data/lib/sidekiq/component.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sidekiq
   ##
   # Sidekiq::Component assumes a config instance is available at @config
@@ -13,7 +15,7 @@ module Sidekiq
 
     def safe_thread(name, &block)
       Thread.new do
-        Thread.current.name = name
+        Thread.current.name = "sidekiq.#{name}"
         watchdog(name, &block)
       end
     end

data/lib/sidekiq/config.rb

@@ -30,7 +30,8 @@ module Sidekiq
       },
       dead_max_jobs: 10_000,
       dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
-      reloader: proc { |&block| block.call }
+      reloader: proc { |&block| block.call },
+      backtrace_cleaner: ->(backtrace) { backtrace }
     }
 
     ERROR_HANDLER = ->(ex, ctx) {
@@ -38,7 +39,10 @@ module Sidekiq
       l = cfg.logger
       l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
       l.warn("#{ex.class.name}: #{ex.message}")
-      l.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
+      unless ex.backtrace.nil?
+        backtrace = cfg[:backtrace_cleaner].call(ex.backtrace)
+        l.warn(backtrace.join("\n"))
+      end
     }
 
     def initialize(options = {})
@@ -52,6 +56,10 @@ module Sidekiq
     def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!
     attr_reader :capsules
 
+    def to_json(*)
+      Sidekiq.dump_json(@options)
+    end
+
     # LEGACY: edits the default capsule
     # config.concurrency = 5
     def concurrency=(val)
@@ -123,7 +131,7 @@ module Sidekiq
     private def local_redis_pool
       # this is our internal client/housekeeping pool. each capsule has its
       # own pool for executing threads.
-      @redis ||= new_redis_pool(5, "internal")
+      @redis ||= new_redis_pool(10, "internal")
     end
 
     def new_redis_pool(size, name = "unset")
@@ -259,7 +267,7 @@ module Sidekiq
       ctx[:_config] = self
       @options[:error_handlers].each do |handler|
         handler.call(ex, ctx)
-      rescue => e
+      rescue Exception => e
         l = logger
         l.error "!!! ERROR HANDLER THREW AN ERROR !!!"
         l.error e

data/lib/sidekiq/embedded.rb

@@ -49,7 +49,7 @@ module Sidekiq
 
           WARNING: Your Redis instance will evict Sidekiq data under heavy load.
           The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
-          See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
+          See: https://github.com/sidekiq/sidekiq/wiki/Using-Redis#memory
 
         EOM
       end

data/lib/sidekiq/fetch.rb

@@ -44,7 +44,7 @@ module Sidekiq # :nodoc:
         return nil
       end
 
-      queue, job = redis { |conn| conn.blocking_call(false, "brpop", *qs, TIMEOUT) }
+      queue, job = redis { |conn| conn.blocking_call(conn.read_timeout + TIMEOUT, "brpop", *qs, TIMEOUT) }
       UnitOfWork.new(queue, job, config) if queue
     end
 

data/lib/sidekiq/job.rb

@@ -239,11 +239,7 @@ module Sidekiq
 
       def perform_bulk(args, batch_size: 1_000)
         client = @klass.build_client
-        result = args.each_slice(batch_size).flat_map do |slice|
-          client.push_bulk(@opts.merge("class" => @klass, "args" => slice))
-        end
-
-        result.is_a?(Enumerator::Lazy) ? result.force : result
+        client.push_bulk(@opts.merge("class" => @klass, "args" => args, :batch_size => batch_size))
       end
 
       # +interval+ must be a timestamp, numeric or something that acts

data/lib/sidekiq/job_retry.rb

@@ -71,6 +71,7 @@ module Sidekiq
     def initialize(capsule)
       @config = @capsule = capsule
       @max_retries = Sidekiq.default_configuration[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
+      @backtrace_cleaner = Sidekiq.default_configuration[:backtrace_cleaner]
     end
 
     # The global retry handler requires only the barest of data.
@@ -159,10 +160,11 @@ module Sidekiq
       end
 
       if msg["backtrace"]
+        backtrace = @backtrace_cleaner.call(exception.backtrace)
         lines = if msg["backtrace"] == true
-          exception.backtrace
+          backtrace
         else
-          exception.backtrace[0...msg["backtrace"].to_i]
+          backtrace[0...msg["backtrace"].to_i]
         end
 
         msg["error_backtrace"] = compress_backtrace(lines)
@@ -171,7 +173,7 @@ module Sidekiq
       # Goodbye dear message, you (re)tried your best I'm sure.
       return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
 
-      strategy, delay = delay_for(jobinst, count, exception)
+      strategy, delay = delay_for(jobinst, count, exception, msg)
       case strategy
       when :discard
         return # poof!
@@ -190,17 +192,18 @@ module Sidekiq
     end
 
     # returns (strategy, seconds)
-    def delay_for(jobinst, count, exception)
+    def delay_for(jobinst, count, exception, msg)
       rv = begin
         # sidekiq_retry_in can return two different things:
         # 1. When to retry next, as an integer of seconds
         # 2. A symbol which re-routes the job elsewhere, e.g. :discard, :kill, :default
-        jobinst&.sidekiq_retry_in_block&.call(count, exception)
+        jobinst&.sidekiq_retry_in_block&.call(count, exception, msg)
       rescue Exception => e
         handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
         nil
       end
 
+      rv = rv.to_i if rv.respond_to?(:to_i)
       delay = (count**4) + 15
       if Integer === rv && rv > 0
         delay = rv

data/lib/sidekiq/job_util.rb

@@ -9,7 +9,7 @@ module Sidekiq
 
     def validate(item)
       raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
-      raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
+      raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array) || item["args"].is_a?(Enumerator::Lazy)
       raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
       raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
       raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
@@ -17,18 +17,23 @@ module Sidekiq
 
     def verify_json(item)
       job_class = item["wrapped"] || item["class"]
-      if Sidekiq::Config::DEFAULTS[:on_complex_arguments] == :raise
-        msg = <<~EOM
-          Job arguments to #{job_class} must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices.
-          To disable this error, add `Sidekiq.strict_args!(false)` to your initializer.
-        EOM
-        raise(ArgumentError, msg) unless json_safe?(item)
-      elsif Sidekiq::Config::DEFAULTS[:on_complex_arguments] == :warn
-        warn <<~EOM unless json_safe?(item)
-          Job arguments to #{job_class} do not serialize to JSON safely. This will raise an error in
-          Sidekiq 7.0. See https://github.com/mperham/sidekiq/wiki/Best-Practices or raise an error today
-          by calling `Sidekiq.strict_args!` during Sidekiq initialization.
-        EOM
+      args = item["args"]
+      mode = Sidekiq::Config::DEFAULTS[:on_complex_arguments]
+
+      if mode == :raise || mode == :warn
+        if (unsafe_item = json_unsafe?(args))
+          msg = <<~EOM
+            Job arguments to #{job_class} must be native JSON types, but #{unsafe_item.inspect} is a #{unsafe_item.class}.
+            See https://github.com/sidekiq/sidekiq/wiki/Best-Practices
+            To disable this error, add `Sidekiq.strict_args!(false)` to your initializer.
+          EOM
+
+          if mode == :raise
+            raise(ArgumentError, msg)
+          else
+            warn(msg)
+          end
+        end
       end
     end
 
@@ -64,8 +69,37 @@ module Sidekiq
 
     private
 
-    def json_safe?(item)
-      JSON.parse(JSON.dump(item["args"])) == item["args"]
+    RECURSIVE_JSON_UNSAFE = {
+      Integer => ->(val) {},
+      Float => ->(val) {},
+      TrueClass => ->(val) {},
+      FalseClass => ->(val) {},
+      NilClass => ->(val) {},
+      String => ->(val) {},
+      Array => ->(val) {
+        val.each do |e|
+          unsafe_item = RECURSIVE_JSON_UNSAFE[e.class].call(e)
+          return unsafe_item unless unsafe_item.nil?
+        end
+        nil
+      },
+      Hash => ->(val) {
+        val.each do |k, v|
+          return k unless String === k
+
+          unsafe_item = RECURSIVE_JSON_UNSAFE[v.class].call(v)
+          return unsafe_item unless unsafe_item.nil?
+        end
+        nil
+      }
+    }
+
+    RECURSIVE_JSON_UNSAFE.default = ->(val) { val }
+    RECURSIVE_JSON_UNSAFE.compare_by_identity
+    private_constant :RECURSIVE_JSON_UNSAFE
+
+    def json_unsafe?(item)
+      RECURSIVE_JSON_UNSAFE[item.class].call(item)
     end
   end
 end

data/lib/sidekiq/launcher.rb

@@ -37,6 +37,7 @@ module Sidekiq
     # and instead have thread call Launcher#heartbeat every N seconds.
     def run(async_beat: true)
       Sidekiq.freeze!
+      logger.debug { @config.merge!({}) }
       @thread = safe_thread("heartbeat", &method(:start_heartbeat)) if async_beat
       @poller.start
       @managers.each(&:start)
@@ -165,7 +166,7 @@ module Sidekiq
           conn.multi { |transaction|
             transaction.sadd("processes", [key])
             transaction.exists(key)
-            transaction.hmset(key, "info", to_json,
+            transaction.hset(key, "info", to_json,
               "busy", curstate.size,
               "beat", Time.now.to_f,
               "rtt_us", rtt,
@@ -214,7 +215,7 @@ module Sidekiq
           Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
           Ensure Redis is running in the same AZ or datacenter as Sidekiq.
           If these values are close to 100,000, that means your Sidekiq process may be
-          CPU-saturated; reduce your concurrency and/or see https://github.com/mperham/sidekiq/discussions/5039
+          CPU-saturated; reduce your concurrency and/or see https://github.com/sidekiq/sidekiq/discussions/5039
         EOM
         RTT_READINGS.reset
       end

data/lib/sidekiq/metrics/query.rb

@@ -70,7 +70,7 @@ module Sidekiq
             result.job_results[klass].add_metric "ms", time, ms.to_i if ms
             result.job_results[klass].add_metric "p", time, p.to_i if p
             result.job_results[klass].add_metric "f", time, f.to_i if f
-            result.job_results[klass].add_hist time, Histogram.new(klass).fetch(conn, time)
+            result.job_results[klass].add_hist time, Histogram.new(klass).fetch(conn, time).reverse
             time -= 60
           end
         end

data/lib/sidekiq/metrics/shared.rb

@@ -29,8 +29,8 @@ module Sidekiq
         1100, 1700, 2500, 3800, 5750,
         8500, 13000, 20000, 30000, 45000,
         65000, 100000, 150000, 225000, 335000,
-        Float::INFINITY # the "maybe your job is too long" bucket
-      ]
+        1e20 # the "maybe your job is too long" bucket
+      ].freeze
       LABELS = [
         "20ms", "30ms", "45ms", "65ms", "100ms",
         "150ms", "225ms", "335ms", "500ms", "750ms",
@@ -38,7 +38,7 @@ module Sidekiq
         "8.5s", "13s", "20s", "30s", "45s",
         "65s", "100s", "150s", "225s", "335s",
         "Slow"
-      ]
+      ].freeze
       FETCH = "GET u16 #0 GET u16 #1 GET u16 #2 GET u16 #3 \
         GET u16 #4 GET u16 #5 GET u16 #6 GET u16 #7 \
         GET u16 #8 GET u16 #9 GET u16 #10 GET u16 #11 \

data/lib/sidekiq/metrics/tracking.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "time"
 require "sidekiq"
 require "sidekiq/metrics/shared"

data/lib/sidekiq/middleware/chain.rb

@@ -166,23 +166,26 @@ module Sidekiq
 
       # Used by Sidekiq to execute the middleware at runtime
       # @api private
-      def invoke(*args)
+      def invoke(*args, &block)
         return yield if empty?
 
         chain = retrieve
-        traverse_chain = proc do
-          if chain.empty?
-            yield
-          else
-            chain.shift.call(*args, &traverse_chain)
+        traverse(chain, 0, args, &block)
+      end
+
+      private
+
+      def traverse(chain, index, args, &block)
+        if index >= chain.size
+          yield
+        else
+          chain[index].call(*args) do
+            traverse(chain, index + 1, args, &block)
           end
         end
-        traverse_chain.call
       end
     end
 
-    private
-
     # Represents each link in the middleware chain
     # @api private
     class Entry