sidekiq 7.3.9 → 8.1.7

data/lib/sidekiq/api.rb

@@ -1,10 +1,8 @@
 # frozen_string_literal: true
 
-require "sidekiq"
-
 require "zlib"
-require "set"
 
+require "sidekiq"
 require "sidekiq/metrics/query"
 
 #
@@ -19,12 +17,37 @@ require "sidekiq/metrics/query"
 #
 
 module Sidekiq
+  module ApiUtils
+    # @api private
+    # Calculate the latency in seconds for a job based on its enqueued timestamp
+    # @param job [Hash] the job hash
+    # @return [Float] latency in seconds
+    def calculate_latency(job)
+      timestamp = job["enqueued_at"] || job["created_at"]
+      return 0.0 unless timestamp
+
+      if timestamp.is_a?(Float)
+        # old format
+        Time.now.to_f - timestamp
+      else
+        now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
+        (now - timestamp) / 1000.0
+      end
+    end
+  end
+
   # Retrieve runtime statistics from Redis regarding
   # this Sidekiq cluster.
   #
   #   stat = Sidekiq::Stats.new
   #   stat.processed
   class Stats
+    QueueSummary = Data.define(:name, :size, :latency, :paused) do
+      alias_method :paused?, :paused
+    end
+
+    include ApiUtils
+
     def initialize
       fetch_stats_fast!
     end
@@ -65,6 +88,7 @@ module Sidekiq
       stat :default_queue_latency
     end
 
+    # @return [Hash{String => Integer}] a hash of queue names to their lengths
     def queues
       Sidekiq.redis do |conn|
         queues = conn.sscan("queues").to_a
@@ -80,6 +104,41 @@ module Sidekiq
       end
     end
 
+    # More detailed information about each queue: name, size, latency, paused status
+    # @return [Array<QueueSummary>]
+    def queue_summaries
+      Sidekiq.redis do |conn|
+        queues = conn.sscan("queues").to_a
+        return [] if queues.empty?
+
+        results = conn.pipelined { |pipeline|
+          queues.each do |queue|
+            pipeline.llen("queue:#{queue}")
+            pipeline.lindex("queue:#{queue}", -1)
+            pipeline.sismember("paused", queue)
+          end
+        }
+
+        queue_summaries = []
+        queues.each_with_index do |name, idx|
+          size = results[idx * 3]
+          last_item = results[idx * 3 + 1]
+          paused = results[idx * 3 + 2] > 0
+
+          latency = if last_item
+            job = Sidekiq.load_json(last_item)
+            calculate_latency(job)
+          else
+            0.0
+          end
+
+          queue_summaries << QueueSummary.new(name:, size:, latency:, paused:)
+        end
+
+        queue_summaries.sort_by { |qd| -qd.size }
+      end
+    end
+
     # O(1) redis calls
     # @api private
     def fetch_stats_fast!
@@ -101,11 +160,10 @@ module Sidekiq
         rescue
           {}
         end
-        now = Time.now.to_f
-        thence = job["enqueued_at"] || now
-        now - thence
+
+        calculate_latency(job)
       else
-        0
+        0.0
       end
 
       @stats = {
@@ -176,7 +234,7 @@ module Sidekiq
     end
 
     class History
-      def initialize(days_previous, start_date = nil, pool: nil)
+      def initialize(days_previous, start_date = nil)
         # we only store five years of data in Redis
         raise ArgumentError if days_previous < 1 || days_previous > (5 * 365)
         @days_previous = days_previous
@@ -226,6 +284,7 @@ module Sidekiq
   #   end
   class Queue
     include Enumerable
+    include ApiUtils
 
     ##
     # Fetch all known queues within Redis.
@@ -236,6 +295,7 @@ module Sidekiq
     end
 
     attr_reader :name
+    alias_method :id, :name
 
     # @param name [String] the name of the queue
     def initialize(name = "default")
@@ -265,11 +325,10 @@ module Sidekiq
       entry = Sidekiq.redis { |conn|
         conn.lindex(@rname, -1)
       }
-      return 0 unless entry
+      return 0.0 unless entry
+
       job = Sidekiq.load_json(entry)
-      now = Time.now.to_f
-      thence = job["enqueued_at"] || now
-      now - thence
+      calculate_latency(job)
     end
 
     def each
@@ -332,6 +391,8 @@ module Sidekiq
   # The job should be considered immutable but may be
   # removed from the queue via JobRecord#delete.
   class JobRecord
+    include ApiUtils
+
     # the parsed Hash of job data
     # @!attribute [r] Item
     attr_reader :item
@@ -417,16 +478,34 @@ module Sidekiq
       self["jid"]
     end
 
+    def iterable_state
+      @iterable_state ||= Sidekiq::IterableJobQuery.new(jid)[jid]
+    end
+
     def bid
       self["bid"]
     end
 
+    def failed_at
+      if self["failed_at"]
+        time_from_timestamp(self["failed_at"])
+      end
+    end
+
+    def retried_at
+      if self["retried_at"]
+        time_from_timestamp(self["retried_at"])
+      end
+    end
+
     def enqueued_at
-      self["enqueued_at"] ? Time.at(self["enqueued_at"]).utc : nil
+      if self["enqueued_at"]
+        time_from_timestamp(self["enqueued_at"])
+      end
     end
 
     def created_at
-      Time.at(self["created_at"] || self["enqueued_at"] || 0).utc
+      time_from_timestamp(self["created_at"] || self["enqueued_at"] || 0)
     end
 
     def tags
@@ -444,8 +523,7 @@ module Sidekiq
     end
 
     def latency
-      now = Time.now.to_f
-      now - (@item["enqueued_at"] || @item["created_at"] || now)
+      calculate_latency(@item)
     end
 
     # Remove this job from the queue
@@ -494,6 +572,15 @@ module Sidekiq
       uncompressed = Zlib::Inflate.inflate(strict_base64_decoded)
       Sidekiq.load_json(uncompressed)
     end
+
+    def time_from_timestamp(timestamp)
+      if timestamp.is_a?(Float)
+        # old format, timestamps were stored as fractional seconds since the epoch
+        Time.at(timestamp).utc
+      else
+        Time.at(timestamp / 1000, timestamp % 1000, :millisecond)
+      end
+    end
   end
 
   # Represents a job within a Redis sorted set where the score
@@ -501,17 +588,24 @@ module Sidekiq
   # could be the scheduled time for it to run (e.g. scheduled set),
   # or the expiration date after which the entry should be deleted (e.g. dead set).
   class SortedEntry < JobRecord
-    attr_reader :score
     attr_reader :parent
 
     # :nodoc:
     # @api private
     def initialize(parent, score, item)
       super(item)
-      @score = Float(score)
+      @score = score
       @parent = parent
     end
 
+    def score
+      Float(@score)
+    end
+
+    def id
+      "#{@score}|#{item["jid"]}"
+    end
+
     # The timestamp associated with this entry
     def at
       Time.at(score).utc
@@ -522,7 +616,7 @@ module Sidekiq
       if @value
         @parent.delete_by_value(@parent.name, @value)
       else
-        @parent.delete_by_jid(score, jid)
+        @parent.delete_by_jid(@score, jid)
       end
     end
 
@@ -531,8 +625,9 @@ module Sidekiq
     # @param at [Time] the new timestamp for this job
     def reschedule(at)
       Sidekiq.redis do |conn|
-        conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
+        conn.zincrby(@parent.name, at.to_f - score, Sidekiq.dump_json(@item))
       end
+      @score = at.to_f
     end
 
     # Enqueue this job from the scheduled or dead set so it will
@@ -567,38 +662,8 @@ module Sidekiq
 
     private
 
-    def remove_job
-      Sidekiq.redis do |conn|
-        results = conn.multi { |transaction|
-          transaction.zrange(parent.name, score, score, "BYSCORE")
-          transaction.zremrangebyscore(parent.name, score, score)
-        }.first
-
-        if results.size == 1
-          yield results.first
-        else
-          # multiple jobs with the same score
-          # find the one with the right JID and push it
-          matched, nonmatched = results.partition { |message|
-            if message.index(jid)
-              msg = Sidekiq.load_json(message)
-              msg["jid"] == jid
-            else
-              false
-            end
-          }
-
-          msg = matched.first
-          yield msg if msg
-
-          # push the rest back onto the sorted set
-          conn.multi do |transaction|
-            nonmatched.each do |message|
-              transaction.zadd(parent.name, score.to_f.to_s, message)
-            end
-          end
-        end
-      end
+    def remove_job(&)
+      parent.remove_job(self, &)
     end
   end
 
@@ -767,6 +832,46 @@ module Sidekiq
       nil
     end
 
+    def remove_job(entry)
+      score = entry.score
+      jid = entry.jid
+      Sidekiq.redis do |conn|
+        results = conn.multi { |transaction|
+          transaction.zrange(name, score, score, "BYSCORE")
+          transaction.zremrangebyscore(name, score, score)
+        }.first
+
+        if results.size == 1
+          yield results.first
+          @_size -= 1
+        else
+          # multiple jobs with the same score
+          # find the one with the right JID and push it
+          matched, nonmatched = results.partition { |message|
+            if message.index(jid)
+              msg = Sidekiq.load_json(message)
+              msg["jid"] == jid
+            else
+              false
+            end
+          }
+
+          msg = matched.first
+          if msg
+            yield msg
+            @_size -= 1
+          end
+
+          # push the rest back onto the sorted set
+          conn.multi do |transaction|
+            nonmatched.each do |message|
+              transaction.zadd(name, score.to_f.to_s, message)
+            end
+          end
+        end
+      end
+    end
+
     # :nodoc:
     # @api private
     def delete_by_value(name, value)
@@ -940,19 +1045,20 @@ module Sidekiq
         # you'll be happier this way
         conn.pipelined do |pipeline|
           procs.each do |key|
-            pipeline.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
+            pipeline.hmget(key, "info", "concurrency", "busy", "beat", "quiet", "rss", "rtt_us")
           end
         end
       }
 
-      result.each do |info, busy, beat, quiet, rss, rtt_us|
+      result.each do |info, concurrency, busy, beat, quiet, rss, rtt_us|
         # If a process is stopped between when we query Redis for `procs` and
         # when we query for `result`, we will have an item in `result` that is
         # composed of `nil` values.
         next if info.nil?
 
         hash = Sidekiq.load_json(info)
-        yield Process.new(hash.merge("busy" => busy.to_i,
+        yield Process.new(hash.merge("concurrency" => concurrency.to_i,
+          "busy" => busy.to_i,
           "beat" => beat.to_f,
           "quiet" => quiet,
           "rss" => rss.to_i,
@@ -1007,9 +1113,9 @@ module Sidekiq
   #   'started_at' => <process start time>,
   #   'pid' => 12345,
   #   'tag' => 'myapp'
-  #   'concurrency' => 25,
-  #   'queues' => ['default', 'low'],
-  #   'busy' => 10,
+  #   'concurrency' => 5,
+  #   'capsules' => {"default" => {"mode" => "weighted", "concurrency" => 5, "weights" => {"default" => 2, "low" => 1}}},
+  #   'busy' => 3,
   #   'beat' => <last heartbeat>,
   #   'identity' => <unique string identifying the process>,
   #   'embedded' => true,
@@ -1036,13 +1142,37 @@ module Sidekiq
     def identity
       self["identity"]
     end
+    alias_method :id, :identity
 
+    # deprecated, use capsules below
     def queues
-      self["queues"]
+      # Backwards compatibility with <8.0.8
+      if !self["capsules"]
+        self["queues"]
+      else
+        capsules.values.flat_map { |x| x["weights"].keys }.uniq
+      end
     end
 
+    # deprecated, use capsules below
     def weights
-      self["weights"]
+      # Backwards compatibility with <8.0.8
+      if !self["capsules"]
+        self["weights"]
+      else
+        hash = {}
+        capsules.values.each do |cap|
+          # Note: will lose data if two capsules are processing the same named queue
+          cap["weights"].each_pair do |queue, weight|
+            hash[queue] = weight
+          end
+        end
+        hash
+      end
+    end
+
+    def capsules
+      self["capsules"]
     end
 
     def version
@@ -1086,6 +1216,10 @@ module Sidekiq
       self["quiet"] == "true"
     end
 
+    def leader?
+      Sidekiq.redis { |c| c.get("dear-leader") == identity }
+    end
+
     private
 
     def signal(sig)
@@ -1114,9 +1248,8 @@ module Sidekiq
   #    works.each do |process_id, thread_id, work|
   #      # process_id is a unique identifier per Sidekiq process
   #      # thread_id is a unique identifier per thread
-  #      # work is a Hash which looks like:
-  #      # { 'queue' => name, 'run_at' => timestamp, 'payload' => job_hash }
-  #      # run_at is an epoch Integer.
+  #      # work is a `Sidekiq::Work` instance that has the following accessor methods.
+  #      # [work.queue, work.run_at, work.payload]
   #    end
   #
   class WorkSet
@@ -1142,7 +1275,7 @@ module Sidekiq
         end
       end
 
-      results.sort_by { |(_, _, hsh)| hsh.raw("run_at") }.each(&block)
+      results.sort_by { |(_, _, work)| work.run_at }.each(&block)
     end
 
     # Note that #size is only as accurate as Sidekiq's heartbeat,
@@ -1172,13 +1305,14 @@ module Sidekiq
     #
     # @param jid [String] the job identifier
     # @return [Sidekiq::Work] the work or nil
-    def find_work_by_jid(jid)
+    def find_work(jid)
       each do |_process_id, _thread_id, work|
         job = work.job
         return work if job.jid == jid
       end
       nil
     end
+    alias_method :find_work_by_jid, :find_work
   end
 
   # Sidekiq::Work represents a job which is currently executing.
@@ -1208,33 +1342,126 @@ module Sidekiq
     def payload
       @hsh["payload"]
     end
+  end
 
-    # deprecated
-    def [](key)
-      kwargs = {uplevel: 1}
-      kwargs[:category] = :deprecated if RUBY_VERSION > "3.0" # TODO
-      warn("Direct access to `Sidekiq::Work` attributes is deprecated, please use `#payload`, `#queue`, `#run_at` or `#job` instead", **kwargs)
+  # Since "worker" is a nebulous term, we've deprecated the use of this class name.
+  # Is "worker" a process, a type of job, a thread? Undefined!
+  # WorkSet better describes the data.
+  Workers = WorkSet
 
-      @hsh[key]
+  class ProfileSet
+    include Enumerable
+
+    # This is a point in time/snapshot API, you'll need to instantiate a new instance
+    # if you want to fetch newer records.
+    def initialize
+      @records = Sidekiq.redis do |c|
+        # This throws away expired profiles
+        c.zremrangebyscore("profiles", "-inf", Time.now.to_f.to_s)
+        # retreive records, newest to oldest
+        c.zrange("profiles", "+inf", 0, "byscore", "rev")
+      end
     end
 
-    # :nodoc:
-    # @api private
-    def raw(name)
-      @hsh[name]
+    def size
+      @records.size
     end
 
-    def method_missing(*all)
-      @hsh.send(*all)
+    def each(&block)
+      fetch_keys = %w[started_at jid type token size elapsed].freeze
+      arrays = Sidekiq.redis do |c|
+        c.pipelined do |p|
+          @records.each do |key|
+            p.hmget(key, *fetch_keys)
+          end
+        end
+      end
+
+      arrays.compact.map { |arr| ProfileRecord.new(arr) }.each(&block)
+    end
+  end
+
+  class ProfileRecord
+    attr_reader :started_at, :jid, :type, :token, :size, :elapsed
+
+    def initialize(arr)
+      # Must be same order as fetch_keys above
+      @started_at = Time.at(Integer(arr[0]))
+      @jid = arr[1]
+      @type = arr[2]
+      @token = arr[3]
+      @size = Integer(arr[4])
+      @elapsed = Float(arr[5])
     end
 
-    def respond_to_missing?(name, *args)
-      @hsh.respond_to?(name)
+    def key
+      "#{token}-#{jid}"
+    end
+
+    def data
+      Sidekiq.redis { |c| c.hget(key, "data") }
     end
   end
 
-  # Since "worker" is a nebulous term, we've deprecated the use of this class name.
-  # Is "worker" a process, a type of job, a thread? Undefined!
-  # WorkSet better describes the data.
-  Workers = WorkSet
+  # Persisted iteration state from Redis for jobs using Sidekiq::IterableJob.
+  class IterableJobQuery
+    def initialize(jids)
+      @cache = bulk_fetch(jids)
+    end
+
+    def [](jid)
+      @cache[jid]
+    end
+
+    private
+
+    # Bulk-fetch iteration state for multiple JIDs in a single Redis pipeline.
+    # Returns a Hash of { jid => State } for JIDs that have iteration state.
+    def bulk_fetch(jids)
+      raise ArgumentError unless jids
+      jids_to_fetch = Array(jids).compact.uniq
+      return {} if jids_to_fetch.empty?
+
+      results = Sidekiq.redis do |conn|
+        conn.pipelined do |pipe|
+          jids_to_fetch.each { |jid| pipe.hgetall("it-#{jid}") }
+        end
+      end
+
+      # TODO Requires Ruby 4
+      # states = ::Hash.new(capacity: jids_to_fetch.size)
+      states = {}
+      jids_to_fetch.each_with_index do |jid, i|
+        raw = results[i]
+        next if raw.nil? || raw.empty?
+
+        states[jid] = State.new(jid, raw)
+      end
+      states
+    end
+
+    State = Struct.new(:jid, :raw) do
+      def executions
+        raw["ex"].to_i
+      end
+
+      def runtime
+        raw["rt"].to_f
+      end
+
+      def cursor
+        @cursor ||= begin
+          Sidekiq.load_json(raw["c"])
+        rescue JSON::ParserError
+          raw["c"]
+        end
+      end
+
+      def cancelled
+        raw["cancelled"]&.to_i
+      end
+    end
+  end
 end
+
+Sidekiq.loader.run_load_hooks(:api)

data/lib/sidekiq/capsule.rb

@@ -11,12 +11,12 @@ module Sidekiq
   # This capsule will pull jobs from the "single" queue and process
   # the jobs with one thread, meaning the jobs will be processed serially.
   #
-  # Sidekiq.configure_server do |config|
-  #   config.capsule("single-threaded") do |cap|
-  #     cap.concurrency = 1
-  #     cap.queues = %w(single)
+  #   Sidekiq.configure_server do |config|
+  #     config.capsule("single-threaded") do |cap|
+  #       cap.concurrency = 1
+  #       cap.queues = %w(single)
+  #     end
   #   end
-  # end
   class Capsule
     include Sidekiq::Component
     extend Forwardable
@@ -27,7 +27,7 @@ module Sidekiq
     attr_reader :mode
     attr_reader :weights
 
-    def_delegators :@config, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
+    def_delegators :@config, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig, :thread_priority
 
     def initialize(name, config)
       @name = name
@@ -38,6 +38,10 @@ module Sidekiq
       @mode = :strict
     end
 
+    def to_h
+      {concurrency: concurrency, mode: mode, weights: weights}
+    end
+
     def fetcher
       @fetcher ||= begin
         instance = (config[:fetch_class] || Sidekiq::BasicFetch).new(self)
@@ -47,7 +51,6 @@ module Sidekiq
     end
 
     def stop
-      fetcher&.bulk_requeue([])
     end
 
     # Sidekiq checks queues in three modes: