sidekiq 7.0.1 → 7.0.8

data/lib/sidekiq/api.rb

@@ -418,6 +418,10 @@ module Sidekiq
       self["jid"]
     end
 
+    def bid
+      self["bid"]
+    end
+
     def enqueued_at
       self["enqueued_at"] ? Time.at(self["enqueued_at"]).utc : nil
     end
@@ -698,7 +702,7 @@ module Sidekiq
     def find_job(jid)
       Sidekiq.redis do |conn|
         conn.zscan(name, match: "*#{jid}*", count: 100) do |entry, score|
-          job = JSON.parse(entry)
+          job = Sidekiq.load_json(entry)
           matched = job["jid"] == jid
           return SortedEntry.new(self, score, entry) if matched
         end
@@ -824,6 +828,24 @@ module Sidekiq
   class ProcessSet
     include Enumerable
 
+    def self.[](identity)
+      exists, (info, busy, beat, quiet, rss, rtt_us) = Sidekiq.redis { |conn|
+        conn.multi { |transaction|
+          transaction.sismember("processes", identity)
+          transaction.hmget(identity, "info", "busy", "beat", "quiet", "rss", "rtt_us")
+        }
+      }
+
+      return nil if exists == 0 || info.nil?
+
+      hash = Sidekiq.load_json(info)
+      Process.new(hash.merge("busy" => busy.to_i,
+        "beat" => beat.to_f,
+        "quiet" => quiet,
+        "rss" => rss.to_i,
+        "rtt_us" => rtt_us.to_i))
+    end
+
     # :nodoc:
     # @api private
     def initialize(clean_plz = true)
@@ -872,7 +894,7 @@ module Sidekiq
         end
       }
 
-      result.each do |info, busy, at_s, quiet, rss, rtt|
+      result.each do |info, busy, beat, quiet, rss, rtt_us|
         # If a process is stopped between when we query Redis for `procs` and
         # when we query for `result`, we will have an item in `result` that is
         # composed of `nil` values.
@@ -880,10 +902,10 @@ module Sidekiq
 
         hash = Sidekiq.load_json(info)
         yield Process.new(hash.merge("busy" => busy.to_i,
-          "beat" => at_s.to_f,
+          "beat" => beat.to_f,
           "quiet" => quiet,
           "rss" => rss.to_i,
-          "rtt_us" => rtt.to_i))
+          "rtt_us" => rtt_us.to_i))
       end
     end
 
@@ -939,6 +961,7 @@ module Sidekiq
   #   'busy' => 10,
   #   'beat' => <last heartbeat>,
   #   'identity' => <unique string identifying the process>,
+  #   'embedded' => true,
   # }
   class Process
     # :nodoc:
@@ -967,11 +990,25 @@ module Sidekiq
       self["queues"]
     end
 
+    def weights
+      self["weights"]
+    end
+
+    def version
+      self["version"]
+    end
+
+    def embedded?
+      self["embedded"]
+    end
+
     # Signal this process to stop processing new jobs.
     # It will continue to execute jobs it has already fetched.
     # This method is *asynchronous* and it can take 5-10
     # seconds for the process to quiet.
     def quiet!
+      raise "Can't quiet an embedded process" if embedded?
+
       signal("TSTP")
     end
 
@@ -980,6 +1017,8 @@ module Sidekiq
     # This method is *asynchronous* and it can take 5-10
     # seconds for the process to start shutting down.
     def stop!
+      raise "Can't stop an embedded process" if embedded?
+
       signal("TERM")
     end
 

data/lib/sidekiq/capsule.rb

@@ -21,12 +21,16 @@ module Sidekiq
     attr_reader :name
     attr_reader :queues
     attr_accessor :concurrency
+    attr_reader :mode
+    attr_reader :weights
 
     def initialize(name, config)
       @name = name
       @config = config
       @queues = ["default"]
+      @weights = {"default" => 0}
       @concurrency = config[:concurrency]
+      @mode = :strict
     end
 
     def fetcher
@@ -41,15 +45,28 @@ module Sidekiq
       fetcher&.bulk_requeue([])
     end
 
+    # Sidekiq checks queues in three modes:
+    # - :strict - all queues have 0 weight and are checked strictly in order
+    # - :weighted - queues have arbitrary weight between 1 and N
+    # - :random - all queues have weight of 1
     def queues=(val)
+      @weights = {}
       @queues = Array(val).each_with_object([]) do |qstr, memo|
         arr = qstr
         arr = qstr.split(",") if qstr.is_a?(String)
         name, weight = arr
+        @weights[name] = weight.to_i
         [weight.to_i, 1].max.times do
           memo << name
         end
       end
+      @mode = if @weights.values.all?(&:zero?)
+        :strict
+      elsif @weights.values.all? { |x| x == 1 }
+        :random
+      else
+        :weighted
+      end
     end
 
     # Allow the middleware to be different per-capsule.

data/lib/sidekiq/cli.rb

@@ -77,13 +77,14 @@ module Sidekiq # :nodoc:
       raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
 
       maxmemory_policy = info["maxmemory_policy"]
-      if maxmemory_policy != "noeviction"
+      if maxmemory_policy != "noeviction" && maxmemory_policy != ""
+        # Redis Enterprise Cloud returns "" for their policy 😳
         logger.warn <<~EOM
 
 
           WARNING: Your Redis instance will evict Sidekiq data under heavy load.
           The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
-          See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
+          See: https://github.com/sidekiq/sidekiq/wiki/Using-Redis#memory
 
         EOM
       end

data/lib/sidekiq/client.rb

@@ -189,7 +189,7 @@ module Sidekiq
       def enqueue_to_in(queue, interval, klass, *args)
         int = interval.to_f
         now = Time.now.to_f
-        ts = (int < 1_000_000_000 ? now + int : int)
+        ts = ((int < 1_000_000_000) ? now + int : int)
 
         item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
         item.delete("at") if ts <= now

data/lib/sidekiq/component.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sidekiq
   ##
   # Sidekiq::Component assumes a config instance is available at @config

data/lib/sidekiq/config.rb

@@ -129,12 +129,12 @@ module Sidekiq
     def new_redis_pool(size, name = "unset")
       # connection pool is lazy, it will not create connections unless you actually need them
       # so don't be skimpy!
-      RedisConnection.create(@redis_config.merge(size: size, logger: logger, pool_name: name))
+      RedisConnection.create({size: size, logger: logger, pool_name: name}.merge(@redis_config))
     end
 
     def redis_info
       redis do |conn|
-        conn.info
+        conn.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
       rescue RedisClientAdapter::CommandError => ex
         # 2850 return fake version when INFO command has (probably) been renamed
         raise unless /unknown command/.match?(ex.message)
@@ -248,7 +248,6 @@ module Sidekiq
         return
       end
 
-      logger.extend(Sidekiq::LoggingUtils)
       @logger = logger
     end
 

data/lib/sidekiq/deploy.rb

@@ -21,15 +21,15 @@ module Sidekiq
     }
 
     def self.mark!(label = nil)
-      label ||= LABEL_MAKER.call
-      Sidekiq::Deploy.new.mark(label: label)
+      Sidekiq::Deploy.new.mark!(label: label)
     end
 
     def initialize(pool = Sidekiq::RedisConnection.create)
       @pool = pool
     end
 
-    def mark(at: Time.now, label: "")
+    def mark!(at: Time.now, label: nil)
+      label ||= LABEL_MAKER.call
       # we need to round the timestamp so that we gracefully
       # handle an very common error in marking deploys:
       # having every process mark its deploy, leading

data/lib/sidekiq/embedded.rb

@@ -49,7 +49,7 @@ module Sidekiq
 
           WARNING: Your Redis instance will evict Sidekiq data under heavy load.
           The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
-          See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
+          See: https://github.com/sidekiq/sidekiq/wiki/Using-Redis#memory
 
         EOM
       end

data/lib/sidekiq/fetch.rb

@@ -30,11 +30,9 @@ module Sidekiq # :nodoc:
     def initialize(cap)
       raise ArgumentError, "missing queue list" unless cap.queues
       @config = cap
-      @strictly_ordered_queues = (config.queues.size == config.queues.uniq.size)
+      @strictly_ordered_queues = cap.mode == :strict
       @queues = config.queues.map { |q| "queue:#{q}" }
-      if @strictly_ordered_queues
-        @queues.uniq!
-      end
+      @queues.uniq! if @strictly_ordered_queues
     end
 
     def retrieve_work
@@ -46,7 +44,7 @@ module Sidekiq # :nodoc:
         return nil
       end
 
-      queue, job = redis { |conn| conn.blocking_call(false, "brpop", *qs, TIMEOUT) }
+      queue, job = redis { |conn| conn.blocking_call(TIMEOUT + 1, "brpop", *qs, TIMEOUT) }
       UnitOfWork.new(queue, job, config) if queue
     end
 

data/lib/sidekiq/job.rb

@@ -258,7 +258,7 @@ module Sidekiq
       def at(interval)
         int = interval.to_f
         now = Time.now.to_f
-        ts = (int < 1_000_000_000 ? now + int : int)
+        ts = ((int < 1_000_000_000) ? now + int : int)
         # Optimization to enqueue something now that is scheduled to go out now or in the past
         @opts["at"] = ts if ts > now
         self
@@ -325,7 +325,7 @@ module Sidekiq
       def perform_in(interval, *args)
         int = interval.to_f
         now = Time.now.to_f
-        ts = (int < 1_000_000_000 ? now + int : int)
+        ts = ((int < 1_000_000_000) ? now + int : int)
 
         item = {"class" => self, "args" => args}
 

data/lib/sidekiq/job_logger.rb

@@ -33,7 +33,7 @@ module Sidekiq
 
       Thread.current[:sidekiq_context] = h
       level = job_hash["log_level"]
-      if level
+      if level && @logger.respond_to?(:log_at)
         @logger.log_at(level, &block)
       else
         yield

data/lib/sidekiq/job_retry.rb

@@ -49,7 +49,7 @@ module Sidekiq
   # The default number of retries is 25 which works out to about 3 weeks
   # You can change the default maximum number of retries in your initializer:
   #
-  #   Sidekiq.options[:max_retries] = 7
+  #   Sidekiq.default_configuration[:max_retries] = 7
   #
   # or limit the number of retries for a particular job and send retries to
   # a low priority queue with:
@@ -171,7 +171,7 @@ module Sidekiq
       # Goodbye dear message, you (re)tried your best I'm sure.
       return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
 
-      strategy, delay = delay_for(jobinst, count, exception)
+      strategy, delay = delay_for(jobinst, count, exception, msg)
       case strategy
       when :discard
         return # poof!
@@ -190,17 +190,18 @@ module Sidekiq
     end
 
     # returns (strategy, seconds)
-    def delay_for(jobinst, count, exception)
+    def delay_for(jobinst, count, exception, msg)
       rv = begin
         # sidekiq_retry_in can return two different things:
         # 1. When to retry next, as an integer of seconds
         # 2. A symbol which re-routes the job elsewhere, e.g. :discard, :kill, :default
-        jobinst&.sidekiq_retry_in_block&.call(count, exception)
+        jobinst&.sidekiq_retry_in_block&.call(count, exception, msg)
       rescue Exception => e
         handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
         nil
       end
 
+      rv = rv.to_i if rv.respond_to?(:to_i)
       delay = (count**4) + 15
       if Integer === rv && rv > 0
         delay = rv

data/lib/sidekiq/job_util.rb

@@ -17,18 +17,23 @@ module Sidekiq
 
     def verify_json(item)
       job_class = item["wrapped"] || item["class"]
-      if Sidekiq::Config::DEFAULTS[:on_complex_arguments] == :raise
-        msg = <<~EOM
-          Job arguments to #{job_class} must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices.
-          To disable this error, add `Sidekiq.strict_args!(false)` to your initializer.
-        EOM
-        raise(ArgumentError, msg) unless json_safe?(item)
-      elsif Sidekiq::Config::DEFAULTS[:on_complex_arguments] == :warn
-        warn <<~EOM unless json_safe?(item)
-          Job arguments to #{job_class} do not serialize to JSON safely. This will raise an error in
-          Sidekiq 7.0. See https://github.com/mperham/sidekiq/wiki/Best-Practices or raise an error today
-          by calling `Sidekiq.strict_args!` during Sidekiq initialization.
-        EOM
+      args = item["args"]
+      mode = Sidekiq::Config::DEFAULTS[:on_complex_arguments]
+
+      if mode == :raise || mode == :warn
+        if (unsafe_item = json_unsafe?(args))
+          msg = <<~EOM
+            Job arguments to #{job_class} must be native JSON types, but #{unsafe_item.inspect} is a #{unsafe_item.class}.
+            See https://github.com/sidekiq/sidekiq/wiki/Best-Practices.
+            To disable this error, add `Sidekiq.strict_args!(false)` to your initializer.
+          EOM
+
+          if mode == :raise
+            raise(ArgumentError, msg)
+          else
+            warn(msg)
+          end
+        end
       end
     end
 
@@ -64,8 +69,37 @@ module Sidekiq
 
     private
 
-    def json_safe?(item)
-      JSON.parse(JSON.dump(item["args"])) == item["args"]
+    RECURSIVE_JSON_UNSAFE = {
+      Integer => ->(val) {},
+      Float => ->(val) {},
+      TrueClass => ->(val) {},
+      FalseClass => ->(val) {},
+      NilClass => ->(val) {},
+      String => ->(val) {},
+      Array => ->(val) {
+        val.each do |e|
+          unsafe_item = RECURSIVE_JSON_UNSAFE[e.class].call(e)
+          return unsafe_item unless unsafe_item.nil?
+        end
+        nil
+      },
+      Hash => ->(val) {
+        val.each do |k, v|
+          return k unless String === k
+
+          unsafe_item = RECURSIVE_JSON_UNSAFE[v.class].call(v)
+          return unsafe_item unless unsafe_item.nil?
+        end
+        nil
+      }
+    }
+
+    RECURSIVE_JSON_UNSAFE.default = ->(val) { val }
+    RECURSIVE_JSON_UNSAFE.compare_by_identity
+    private_constant :RECURSIVE_JSON_UNSAFE
+
+    def json_unsafe?(item)
+      RECURSIVE_JSON_UNSAFE[item.class].call(item)
     end
   end
 end

data/lib/sidekiq/launcher.rb

@@ -37,6 +37,7 @@ module Sidekiq
     # and instead have thread call Launcher#heartbeat every N seconds.
     def run(async_beat: true)
       Sidekiq.freeze!
+      logger.debug { @config.merge!({}) }
       @thread = safe_thread("heartbeat", &method(:start_heartbeat)) if async_beat
       @poller.start
       @managers.each(&:start)
@@ -161,7 +162,7 @@ module Sidekiq
         fails = procd = 0
         kb = memory_usage(::Process.pid)
 
-        _, exists, _, _, msg = redis { |conn|
+        _, exists, _, _, signal = redis { |conn|
           conn.multi { |transaction|
             transaction.sadd("processes", [key])
             transaction.exists(key)
@@ -180,9 +181,7 @@ module Sidekiq
         fire_event(:heartbeat) unless exists > 0
         fire_event(:beat, oneshot: false)
 
-        return unless msg
-
-        ::Process.kill(msg, ::Process.pid)
+        ::Process.kill(signal, ::Process.pid) if signal && !@embedded
       rescue => e
         # ignore all redis/network issues
         logger.error("heartbeat: #{e}")
@@ -216,7 +215,7 @@ module Sidekiq
           Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
           Ensure Redis is running in the same AZ or datacenter as Sidekiq.
           If these values are close to 100,000, that means your Sidekiq process may be
-          CPU-saturated; reduce your concurrency and/or see https://github.com/mperham/sidekiq/discussions/5039
+          CPU-saturated; reduce your concurrency and/or see https://github.com/sidekiq/sidekiq/discussions/5039
         EOM
         RTT_READINGS.reset
       end
@@ -250,12 +249,19 @@ module Sidekiq
         "pid" => ::Process.pid,
         "tag" => @config[:tag] || "",
         "concurrency" => @config.total_concurrency,
-        "queues" => @config.capsules.values.map { |cap| cap.queues }.flatten.uniq,
+        "queues" => @config.capsules.values.flat_map { |cap| cap.queues }.uniq,
+        "weights" => to_weights,
         "labels" => @config[:labels].to_a,
-        "identity" => identity
+        "identity" => identity,
+        "version" => Sidekiq::VERSION,
+        "embedded" => @embedded
       }
     end
 
+    def to_weights
+      @config.capsules.values.map(&:weights)
+    end
+
     def to_json
       # this data changes infrequently so dump it to a string
       # now so we don't need to dump it every heartbeat.

data/lib/sidekiq/metrics/query.rb

@@ -123,7 +123,7 @@ module Sidekiq
         def series_avg(metric = "ms")
           series[metric].each_with_object(Hash.new(0)) do |(bucket, value), result|
             completed = series.dig("p", bucket) - series.dig("f", bucket)
-            result[bucket] = completed == 0 ? 0 : value.to_f / completed
+            result[bucket] = (completed == 0) ? 0 : value.to_f / completed
           end
         end
       end

data/lib/sidekiq/metrics/tracking.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "time"
 require "sidekiq"
 require "sidekiq/metrics/shared"

data/lib/sidekiq/middleware/chain.rb

@@ -166,23 +166,26 @@ module Sidekiq
 
       # Used by Sidekiq to execute the middleware at runtime
       # @api private
-      def invoke(*args)
+      def invoke(*args, &block)
         return yield if empty?
 
         chain = retrieve
-        traverse_chain = proc do
-          if chain.empty?
-            yield
-          else
-            chain.shift.call(*args, &traverse_chain)
+        traverse(chain, 0, args, &block)
+      end
+
+      private
+
+      def traverse(chain, index, args, &block)
+        if index >= chain.size
+          yield
+        else
+          chain[index].call(*args) do
+            traverse(chain, index + 1, args, &block)
           end
         end
-        traverse_chain.call
       end
     end
 
-    private
-
     # Represents each link in the middleware chain
     # @api private
     class Entry

data/lib/sidekiq/middleware/current_attributes.rb

@@ -22,13 +22,11 @@ module Sidekiq
       end
 
       def call(_, job, _, _)
-        attrs = @strklass.constantize.attributes
-        if attrs.any?
-          if job.has_key?("cattr")
-            job["cattr"].merge!(attrs)
-          else
-            job["cattr"] = attrs
-          end
+        if !job.has_key?("cattr")
+          attrs = @strklass.constantize.attributes
+          # Retries can push the job N times, we don't
+          # want retries to reset cattr. #5692, #5090
+          job["cattr"] = attrs if attrs.any?
         end
         yield
       end

data/lib/sidekiq/monitor.rb

@@ -16,8 +16,6 @@ class Sidekiq::Monitor
         return
       end
       send(section)
-    rescue => e
-      abort "Couldn't get status: #{e}"
     end
 
     def all
@@ -49,10 +47,25 @@ class Sidekiq::Monitor
     def processes
       puts "---- Processes (#{process_set.size}) ----"
       process_set.each_with_index do |process, index|
+        # Keep compatibility with legacy versions since we don't want to break sidekiqmon during rolling upgrades or downgrades.
+        #
+        # Before:
+        #   ["default", "critical"]
+        #
+        # After:
+        #   {"default" => 1, "critical" => 10}
+        queues =
+          if process["weights"]
+            process["weights"].sort_by { |queue| queue[0] }.map { |capsule| capsule.map { |name, weight| (weight > 0) ? "#{name}: #{weight}" : name }.join(", ") }
+          else
+            process["queues"].sort
+          end
+
         puts "#{process["identity"]} #{tags_for(process)}"
         puts "  Started: #{Time.at(process["started_at"])} (#{time_ago(process["started_at"])})"
         puts "  Threads: #{process["concurrency"]} (#{process["busy"]} busy)"
-        puts "   Queues: #{split_multiline(process["queues"].sort, pad: 11)}"
+        puts "   Queues: #{split_multiline(queues, pad: 11)}"
+        puts "  Version: #{process["version"] || "Unknown"}" if process["version"] != Sidekiq::VERSION
         puts "" unless (index + 1) == process_set.size
       end
     end
@@ -101,7 +114,7 @@ class Sidekiq::Monitor
       tags = [
         process["tag"],
         process["labels"],
-        (process["quiet"] == "true" ? "quiet" : nil)
+        ((process["quiet"] == "true") ? "quiet" : nil)
       ].flatten.compact
       tags.any? ? "[#{tags.join("] [")}]" : nil
     end

data/lib/sidekiq/paginator.rb

@@ -3,7 +3,7 @@
 module Sidekiq
   module Paginator
     def page(key, pageidx = 1, page_size = 25, opts = nil)
-      current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
+      current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
       pageidx = current_page - 1
       total_size = 0
       items = []
@@ -45,7 +45,7 @@ module Sidekiq
     end
 
     def page_items(items, pageidx = 1, page_size = 25)
-      current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
+      current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
       pageidx = current_page - 1
       starting = pageidx * page_size
       items = items.to_a

data/lib/sidekiq/processor.rb

@@ -146,6 +146,9 @@ module Sidekiq
       end
     end
 
+    IGNORE_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :never}
+    private_constant :IGNORE_SHUTDOWN_INTERRUPTS
+
     def process(uow)
       jobstr = uow.job
       queue = uow.queue_name
@@ -195,7 +198,7 @@ module Sidekiq
       ensure
         if ack
           # We don't want a shutdown signal to interrupt job acknowledgment.
-          Thread.handle_interrupt(Sidekiq::Shutdown => :never) do
+          Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) do
             uow.acknowledge
           end
         end

data/lib/sidekiq/rails.rb

@@ -11,7 +11,8 @@ module Sidekiq
       end
 
       def call
-        @app.reloader.wrap do
+        params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
+        @app.reloader.wrap(**params) do
           yield
         end
       end

data/lib/sidekiq/redis_client_adapter.rb

@@ -9,10 +9,12 @@ module Sidekiq
     CommandError = RedisClient::CommandError
 
     module CompatMethods
+      # TODO Deprecate and remove this
       def info
         @client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
       end
 
+      # TODO Deprecate and remove this
       def evalsha(sha, keys, argv)
         @client.call("EVALSHA", sha, keys.size, *keys, *argv)
       end
@@ -34,12 +36,7 @@ module Sidekiq
     CompatClient = RedisClient::Decorator.create(CompatMethods)
 
     class CompatClient
-      # underscore methods are not official API
-      def _client
-        @client
-      end
-
-      def _config
+      def config
         @client.config
       end