sidekiq 7.3.10 → 8.0.10

data/lib/sidekiq/api.rb

@@ -1,10 +1,8 @@
 # frozen_string_literal: true
 
-require "sidekiq"
-
 require "zlib"
-require "set"
 
+require "sidekiq"
 require "sidekiq/metrics/query"
 
 #
@@ -101,11 +99,22 @@ module Sidekiq
         rescue
           {}
         end
-        now = Time.now.to_f
-        thence = job["enqueued_at"] || now
-        now - thence
+
+        enqueued_at = job["enqueued_at"]
+        if enqueued_at
+          if enqueued_at.is_a?(Float)
+            # old format
+            now = Time.now.to_f
+            now - enqueued_at
+          else
+            now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
+            (now - enqueued_at) / 1000.0
+          end
+        else
+          0.0
+        end
       else
-        0
+        0.0
       end
 
       @stats = {
@@ -265,11 +274,22 @@ module Sidekiq
       entry = Sidekiq.redis { |conn|
         conn.lindex(@rname, -1)
       }
-      return 0 unless entry
+      return 0.0 unless entry
+
       job = Sidekiq.load_json(entry)
-      now = Time.now.to_f
-      thence = job["enqueued_at"] || now
-      now - thence
+      enqueued_at = job["enqueued_at"]
+      if enqueued_at
+        if enqueued_at.is_a?(Float)
+          # old format
+          now = Time.now.to_f
+          now - enqueued_at
+        else
+          now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
+          (now - enqueued_at) / 1000.0
+        end
+      else
+        0.0
+      end
     end
 
     def each
@@ -421,12 +441,26 @@ module Sidekiq
       self["bid"]
     end
 
+    def failed_at
+      if self["failed_at"]
+        time_from_timestamp(self["failed_at"])
+      end
+    end
+
+    def retried_at
+      if self["retried_at"]
+        time_from_timestamp(self["retried_at"])
+      end
+    end
+
     def enqueued_at
-      self["enqueued_at"] ? Time.at(self["enqueued_at"]).utc : nil
+      if self["enqueued_at"]
+        time_from_timestamp(self["enqueued_at"])
+      end
     end
 
     def created_at
-      Time.at(self["created_at"] || self["enqueued_at"] || 0).utc
+      time_from_timestamp(self["created_at"] || self["enqueued_at"] || 0)
     end
 
     def tags
@@ -444,8 +478,17 @@ module Sidekiq
     end
 
     def latency
-      now = Time.now.to_f
-      now - (@item["enqueued_at"] || @item["created_at"] || now)
+      timestamp = @item["enqueued_at"] || @item["created_at"]
+      if timestamp
+        if timestamp.is_a?(Float)
+          # old format
+          Time.now.to_f - timestamp
+        else
+          (::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) - timestamp) / 1000.0
+        end
+      else
+        0.0
+      end
     end
 
     # Remove this job from the queue
@@ -494,6 +537,15 @@ module Sidekiq
       uncompressed = Zlib::Inflate.inflate(strict_base64_decoded)
       Sidekiq.load_json(uncompressed)
     end
+
+    def time_from_timestamp(timestamp)
+      if timestamp.is_a?(Float)
+        # old format, timestamps were stored as fractional seconds since the epoch
+        Time.at(timestamp).utc
+      else
+        Time.at(timestamp / 1000, timestamp % 1000, :millisecond)
+      end
+    end
   end
 
   # Represents a job within a Redis sorted set where the score
@@ -1007,9 +1059,9 @@ module Sidekiq
   #   'started_at' => <process start time>,
   #   'pid' => 12345,
   #   'tag' => 'myapp'
-  #   'concurrency' => 25,
-  #   'queues' => ['default', 'low'],
-  #   'busy' => 10,
+  #   'concurrency' => 5,
+  #   'capsules' => {"default" => {"mode" => "weighted", "concurrency" => 5, "weights" => {"default" => 2, "low" => 1}}},
+  #   'busy' => 3,
   #   'beat' => <last heartbeat>,
   #   'identity' => <unique string identifying the process>,
   #   'embedded' => true,
@@ -1037,12 +1089,35 @@ module Sidekiq
       self["identity"]
     end
 
+    # deprecated, use capsules below
     def queues
-      self["queues"]
+      # Backwards compatibility with <8.0.8
+      if !self["capsules"]
+        self["queues"]
+      else
+        capsules.values.flat_map { |x| x["weights"].keys }.uniq
+      end
     end
 
+    # deprecated, use capsules below
     def weights
-      self["weights"]
+      # Backwards compatibility with <8.0.8
+      if !self["capsules"]
+        self["weights"]
+      else
+        hash = {}
+        capsules.values.each do |cap|
+          # Note: will lose data if two capsules are processing the same named queue
+          cap["weights"].each_pair do |queue, weight|
+            hash[queue] = weight
+          end
+        end
+        hash
+      end
+    end
+
+    def capsules
+      self["capsules"]
     end
 
     def version
@@ -1114,9 +1189,8 @@ module Sidekiq
   #    works.each do |process_id, thread_id, work|
   #      # process_id is a unique identifier per Sidekiq process
   #      # thread_id is a unique identifier per thread
-  #      # work is a Hash which looks like:
-  #      # { 'queue' => name, 'run_at' => timestamp, 'payload' => job_hash }
-  #      # run_at is an epoch Integer.
+  #      # work is a `Sidekiq::Work` instance that has the following accessor methods.
+  #      # [work.queue, work.run_at, work.payload]
   #    end
   #
   class WorkSet
@@ -1142,7 +1216,7 @@ module Sidekiq
         end
       end
 
-      results.sort_by { |(_, _, hsh)| hsh.raw("run_at") }.each(&block)
+      results.sort_by { |(_, _, work)| work.run_at }.each(&block)
     end
 
     # Note that #size is only as accurate as Sidekiq's heartbeat,
@@ -1172,13 +1246,14 @@ module Sidekiq
     #
     # @param jid [String] the job identifier
     # @return [Sidekiq::Work] the work or nil
-    def find_work_by_jid(jid)
+    def find_work(jid)
       each do |_process_id, _thread_id, work|
         job = work.job
         return work if job.jid == jid
       end
       nil
     end
+    alias_method :find_work_by_jid, :find_work
   end
 
   # Sidekiq::Work represents a job which is currently executing.
@@ -1208,33 +1283,66 @@ module Sidekiq
     def payload
       @hsh["payload"]
     end
+  end
 
-    # deprecated
-    def [](key)
-      kwargs = {uplevel: 1}
-      kwargs[:category] = :deprecated if RUBY_VERSION > "3.0" # TODO
-      warn("Direct access to `Sidekiq::Work` attributes is deprecated, please use `#payload`, `#queue`, `#run_at` or `#job` instead", **kwargs)
+  # Since "worker" is a nebulous term, we've deprecated the use of this class name.
+  # Is "worker" a process, a type of job, a thread? Undefined!
+  # WorkSet better describes the data.
+  Workers = WorkSet
 
-      @hsh[key]
-    end
+  class ProfileSet
+    include Enumerable
 
-    # :nodoc:
-    # @api private
-    def raw(name)
-      @hsh[name]
+    # This is a point in time/snapshot API, you'll need to instantiate a new instance
+    # if you want to fetch newer records.
+    def initialize
+      @records = Sidekiq.redis do |c|
+        # This throws away expired profiles
+        c.zremrangebyscore("profiles", "-inf", Time.now.to_f.to_s)
+        # retreive records, newest to oldest
+        c.zrange("profiles", "+inf", 0, "byscore", "rev")
+      end
     end
 
-    def method_missing(*all)
-      @hsh.send(*all)
+    def size
+      @records.size
     end
 
-    def respond_to_missing?(name, *args)
-      @hsh.respond_to?(name)
+    def each(&block)
+      fetch_keys = %w[started_at jid type token size elapsed].freeze
+      arrays = Sidekiq.redis do |c|
+        c.pipelined do |p|
+          @records.each do |key|
+            p.hmget(key, *fetch_keys)
+          end
+        end
+      end
+
+      arrays.compact.map { |arr| ProfileRecord.new(arr) }.each(&block)
     end
   end
 
-  # Since "worker" is a nebulous term, we've deprecated the use of this class name.
-  # Is "worker" a process, a type of job, a thread? Undefined!
-  # WorkSet better describes the data.
-  Workers = WorkSet
+  class ProfileRecord
+    attr_reader :started_at, :jid, :type, :token, :size, :elapsed
+
+    def initialize(arr)
+      # Must be same order as fetch_keys above
+      @started_at = Time.at(Integer(arr[0]))
+      @jid = arr[1]
+      @type = arr[2]
+      @token = arr[3]
+      @size = Integer(arr[4])
+      @elapsed = Float(arr[5])
+    end
+
+    def key
+      "#{token}-#{jid}"
+    end
+
+    def data
+      Sidekiq.redis { |c| c.hget(key, "data") }
+    end
+  end
 end
+
+Sidekiq.loader.run_load_hooks(:api)

data/lib/sidekiq/capsule.rb

@@ -11,12 +11,12 @@ module Sidekiq
   # This capsule will pull jobs from the "single" queue and process
   # the jobs with one thread, meaning the jobs will be processed serially.
   #
-  # Sidekiq.configure_server do |config|
-  #   config.capsule("single-threaded") do |cap|
-  #     cap.concurrency = 1
-  #     cap.queues = %w(single)
+  #   Sidekiq.configure_server do |config|
+  #     config.capsule("single-threaded") do |cap|
+  #       cap.concurrency = 1
+  #       cap.queues = %w(single)
+  #     end
   #   end
-  # end
   class Capsule
     include Sidekiq::Component
     extend Forwardable
@@ -27,7 +27,7 @@ module Sidekiq
     attr_reader :mode
     attr_reader :weights
 
-    def_delegators :@config, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
+    def_delegators :@config, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig, :thread_priority
 
     def initialize(name, config)
       @name = name
@@ -38,6 +38,10 @@ module Sidekiq
       @mode = :strict
     end
 
+    def to_h
+      {concurrency: concurrency, mode: mode, weights: weights}
+    end
+
     def fetcher
       @fetcher ||= begin
         instance = (config[:fetch_class] || Sidekiq::BasicFetch).new(self)

data/lib/sidekiq/cli.rb

@@ -3,7 +3,6 @@
 $stdout.sync = true
 
 require "yaml"
-require "singleton"
 require "optparse"
 require "erb"
 require "fileutils"
@@ -17,7 +16,6 @@ require "sidekiq/launcher"
 module Sidekiq # :nodoc:
   class CLI
     include Sidekiq::Component
-    include Singleton unless $TESTING
 
     attr_accessor :launcher
     attr_accessor :environment
@@ -31,6 +29,10 @@ module Sidekiq # :nodoc:
       validate!
     end
 
+    def self.instance
+      @instance ||= new
+    end
+
     def jruby?
       defined?(::JRUBY_VERSION)
     end
@@ -47,7 +49,7 @@ module Sidekiq # :nodoc:
       logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
 
       self_read, self_write = IO.pipe
-      sigs = %w[INT TERM TTIN TSTP]
+      sigs = %w[INT TERM INFO TTIN TSTP]
       # USR1 and USR2 don't work on the JVM
       sigs << "USR2" if Sidekiq.pro? && !jruby?
       sigs.each do |sig|
@@ -74,7 +76,7 @@ module Sidekiq # :nodoc:
       # fire startup and start multithreading.
       info = @config.redis_info
       ver = Gem::Version.new(info["redis_version"])
-      raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
+      raise "You are connected to Redis #{ver}, Sidekiq requires Redis 7.0.0 or greater" if ver < Gem::Version.new("7.0.0")
 
       maxmemory_policy = info["maxmemory_policy"]
       if maxmemory_policy != "noeviction" && maxmemory_policy != ""
@@ -199,6 +201,7 @@ module Sidekiq # :nodoc:
         cli.logger.info "Received TSTP, no longer accepting new work"
         cli.launcher.quiet
       },
+      # deprecated, use INFO
       "TTIN" => ->(cli) {
         Thread.list.each do |thread|
           cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
@@ -208,14 +211,24 @@ module Sidekiq # :nodoc:
             cli.logger.warn "<no backtrace available>"
           end
         end
+      },
+      "INFO" => ->(cli) {
+        Thread.list.each do |thread|
+          cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
+          if thread.backtrace
+            cli.logger.warn thread.backtrace.join("\n")
+          else
+            cli.logger.warn "<no backtrace available>"
+          end
+        end
       }
     }
-    UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
-    SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
 
     def handle_signal(sig)
       logger.debug "Got #{sig} signal"
-      SIGNAL_HANDLERS[sig].call(self)
+      hndlr = SIGNAL_HANDLERS[sig]
+      hndlr ? hndlr.call(self) :
+        logger.warn("No #{sig} signal handler registered, ignoring")
     end
 
     private
@@ -298,29 +311,18 @@ module Sidekiq # :nodoc:
 
       if File.directory?(@config[:require])
         require "rails"
-        if ::Rails::VERSION::MAJOR < 6
-          warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 6+"
+        if ::Rails::VERSION::MAJOR < 7
+          warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 7+"
         end
         require "sidekiq/rails"
         require File.expand_path("#{@config[:require]}/config/environment.rb")
-        @config[:tag] ||= default_tag
+        @config[:tag] ||= default_tag(::Rails.root)
       else
         require @config[:require]
+        @config[:tag] ||= default_tag
       end
     end
 
-    def default_tag
-      dir = ::Rails.root
-      name = File.basename(dir)
-      prevdir = File.dirname(dir) # Capistrano release directory?
-      if name.to_i != 0 && prevdir
-        if File.basename(prevdir) == "releases"
-          return File.basename(File.dirname(prevdir))
-        end
-      end
-      name
-    end
-
     def validate!
       if !File.exist?(@config[:require]) ||
           (File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
@@ -395,7 +397,12 @@ module Sidekiq # :nodoc:
     end
 
     def initialize_logger
-      @config.logger.level = ::Logger::DEBUG if @config[:verbose]
+      if @config[:verbose] || ENV["DEBUG_INVOCATION"] == "1"
+        # DEBUG_INVOCATION is a systemd-ism triggered by
+        # RestartMode=debug. We turn on debugging when the
+        # sidekiq process crashes and is restarted with this flag.
+        @config.logger.level = ::Logger::DEBUG
+      end
     end
 
     def parse_config(path)

data/lib/sidekiq/client.rb

@@ -67,9 +67,7 @@ module Sidekiq
         c.pipelined do |p|
           p.hsetnx(key, "cancelled", Time.now.to_i)
           p.hget(key, "cancelled")
-          p.expire(key, Sidekiq::Job::Iterable::STATE_TTL)
-          # TODO When Redis 7.2 is required
-          # p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
+          p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
         end
       end
       result.to_i
@@ -119,6 +117,9 @@ module Sidekiq
     # larger than 1000 but YMMV based on network quality, size of job args, etc.
     # A large number of jobs can cause a bit of Redis command processing latency.
     #
+    # Accepts an additional `:spread_interval` option (in seconds) to randomly spread
+    # the jobs schedule times over the specified interval.
+    #
     # Takes the same arguments as #push except that args is expected to be
     # an Array of Arrays.  All other keys are duplicated for each job.  Each job
     # is run through the client middleware pipeline and each job gets its own Job ID
@@ -133,13 +134,24 @@ module Sidekiq
     def push_bulk(items)
       batch_size = items.delete(:batch_size) || items.delete("batch_size") || 1_000
       args = items["args"]
-      at = items.delete("at")
+      at = items.delete("at") || items.delete(:at)
       raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
       raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
       jid = items.delete("jid")
       raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
 
+      spread_interval = items.delete(:spread_interval) || items.delete("spread_interval")
+      raise ArgumentError, "Jobs 'spread_interval' must be a positive Numeric" if spread_interval && (!spread_interval.is_a?(Numeric) || spread_interval <= 0)
+      raise ArgumentError, "Only one of 'at' or 'spread_interval' can be provided" if at && spread_interval
+
+      if !at && spread_interval
+        # Do not use spread interval smaller than pooling interval.
+        spread_interval = [spread_interval, 5].max
+        now = Time.now.to_f
+        at = args.map { now + rand * spread_interval }
+      end
+
       normed = normalize_item(items)
       slice_index = 0
       result = args.each_slice(batch_size).flat_map do |slice|
@@ -266,22 +278,21 @@ module Sidekiq
       if payloads.first.key?("at")
         conn.zadd("schedule", payloads.flat_map { |hash|
           at = hash["at"].to_s
-          # ActiveJob sets this but the job has not been enqueued yet
-          hash.delete("enqueued_at")
-          # TODO: Use hash.except("at") when support for Ruby 2.7 is dropped
-          hash = hash.dup
-          hash.delete("at")
+          # ActiveJob sets enqueued_at but the job has not been enqueued yet
+          hash = hash.except("enqueued_at", "at")
           [at, Sidekiq.dump_json(hash)]
         })
       else
-        queue = payloads.first["queue"]
-        now = Time.now.to_f
-        to_push = payloads.map { |entry|
-          entry["enqueued_at"] = now
-          Sidekiq.dump_json(entry)
-        }
-        conn.sadd("queues", [queue])
-        conn.lpush("queue:#{queue}", to_push)
+        now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) # milliseconds since the epoch
+        grouped_queues = payloads.group_by { |job| job["queue"] }
+        conn.sadd("queues", grouped_queues.keys)
+        grouped_queues.each do |queue, grouped_payloads|
+          to_push = grouped_payloads.map { |entry|
+            entry["enqueued_at"] = now
+            Sidekiq.dump_json(entry)
+          }
+          conn.lpush("queue:#{queue}", to_push)
+        end
       end
     end
   end

data/lib/sidekiq/component.rb

@@ -1,11 +1,39 @@
 # frozen_string_literal: true
 
 module Sidekiq
+  # Ruby's default thread priority is 0, which uses 100ms time slices.
+  # This can lead to some surprising thread starvation; if using a lot of
+  # CPU-heavy concurrency, it may take several seconds before a Thread gets
+  # on the CPU.
+  #
+  # Negative priorities lower the timeslice by half, so -1 = 50ms, -2 = 25ms, etc.
+  # With more frequent timeslices, we reduce the risk of unintentional timeouts
+  # and starvation.
+  #
+  # Customize like so:
+  #
+  #   Sidekiq.configure_server do |cfg|
+  #     cfg.thread_priority = 0
+  #   end
+  #
+  DEFAULT_THREAD_PRIORITY = -1
+
   ##
-  # Sidekiq::Component assumes a config instance is available at @config
+  # Sidekiq::Component provides a set of utility methods depending only
+  # on Sidekiq::Config. It assumes a config instance is available at @config.
   module Component # :nodoc:
     attr_reader :config
 
+    # This is epoch milliseconds, appropriate for persistence
+    def real_ms
+      ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
+    end
+
+    # used for time difference and relative comparisons, not persistence.
+    def mono_ms
+      ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
+    end
+
     def watchdog(last_words)
       yield
     rescue Exception => ex
@@ -13,11 +41,11 @@ module Sidekiq
       raise ex
     end
 
-    def safe_thread(name, &block)
+    def safe_thread(name, priority: nil, &block)
       Thread.new do
         Thread.current.name = "sidekiq.#{name}"
         watchdog(name, &block)
-      end
+      end.tap { |t| t.priority = (priority || config.thread_priority || DEFAULT_THREAD_PRIORITY) }
     end
 
     def logger
@@ -86,5 +114,16 @@ module Sidekiq
         end.join(", ")
       }>"
     end
+
+    def default_tag(dir = Dir.pwd)
+      name = File.basename(dir)
+      prevdir = File.dirname(dir) # Capistrano release directory?
+      if name.to_i != 0 && prevdir
+        if File.basename(prevdir) == "releases"
+          return File.basename(File.dirname(prevdir))
+        end
+      end
+      name
+    end
   end
 end