sidekiq 6.5.0 → 6.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dedc78179d612e64435e57f0f6644c2645c06ffdc24151c9ad51df4035b51efa
-  data.tar.gz: 5987eba09190793cfccbff56066b54b5308a1fe74be7551cb5b9d6d03d256c3a
+  metadata.gz: 46b428cecc373a6b11d27e12622dcfc8661b138e244c16575ad49362c672a60e
+  data.tar.gz: 64fc302563e615eec63e8701b581949cb90738c799714257d98bbd6f359cff45
 SHA512:
-  metadata.gz: 9a26b5eb6e2248870f07ede054cac8330b52ad548d27b40d3950b6d985ef5ef8359e4dfc062757dc4b479036f0b2716bacfc821a20bd63413ea7c3f772bc6ead
-  data.tar.gz: 46a35f7f27693364b1b6077e602d3f69e476508654157f48fcd457a70bbcdb563f8515812ef37abae207604596e1d42f78132ac54471109f913e87c11a707cdb
+  metadata.gz: 6b62cefbc386b0113ffe47ca4e10b6752cfd86d973c0a8143e116ad45e2e6c680f0de686c9e2750760c14c71f17e3c13375145ae16a98f6cceab96908ca4cf61
+  data.tar.gz: 0dfc564242453b670e13e656098a3294e0c9831efa9520e025b08d3c265ef6cb148fb4519af78406a43d9972b9e83db1c9d8271ee7288f57401a5e7bb30d401e

data/Changes.md

@@ -2,12 +2,32 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
 
+6.5.3
+----------
+
+- Don't require metrics code without explicit opt-in [#5456]
+
+6.5.2
+----------
+
+- [Job Metrics are under active development, help wanted!](https://github.com/mperham/sidekiq/wiki/Metrics#contributing) **BETA**
+- Add `Context` column on queue page which shows any CurrentAttributes [#5450]
+- `sidekiq_retry_in` may now return `:discard` or `:kill` to dynamically stop job retries [#5406]
+- Smarter sorting of processes in /busy Web UI [#5398]
+- Fix broken hamburger menu in mobile UI [#5428]
+- Require redis-rb 4.5.0. Note that Sidekiq will break if you use the
+  [`Redis.exists_returns_integer = false`](https://github.com/redis/redis-rb/blob/master/CHANGELOG.md#450) flag. [#5394]
+
+6.5.1
+----------
+
+- Fix `push_bulk` breakage [#5387]
+
 6.5.0
 ---------
 
 - Substantial refactoring of Sidekiq server internals, part of a larger effort
-  to reduce Sidekiq's internal usage of global methods and data, see [docs/component.md](docs/component.md),
-  [docs/global_to_local.md](docs/global_to_local.md) and [docs/middleware.md](docs/middleware.md).
+  to reduce Sidekiq's internal usage of global methods and data, see [docs/global_to_local.md](docs/global_to_local.md) and [docs/middleware.md](docs/middleware.md).
 - **Add beta support for the `redis-client` gem**. This will become the default Redis driver in Sidekiq 7.0. [#5298]
   Read more: https://github.com/mperham/sidekiq/wiki/Using-redis-client
 - **Add beta support for DB transaction-aware client** [#5291]

data/lib/sidekiq/api.rb

@@ -3,9 +3,17 @@
 require "sidekiq"
 
 require "zlib"
+require "set"
 require "base64"
+require "sidekiq/metrics/deploy"
+require "sidekiq/metrics/query"
 
 module Sidekiq
+  # Retrieve runtime statistics from Redis regarding
+  # this Sidekiq cluster.
+  #
+  #   stat = Sidekiq::Stats.new
+  #   stat.processed
   class Stats
     def initialize
       fetch_stats_fast!
@@ -52,6 +60,7 @@ module Sidekiq
     end
 
     # O(1) redis calls
+    # @api private
     def fetch_stats_fast!
       pipe1_res = Sidekiq.redis { |conn|
         conn.pipelined do |pipeline|
@@ -91,6 +100,7 @@ module Sidekiq
     end
 
     # O(number of processes + number of queues) redis calls
+    # @api private
     def fetch_stats_slow!
       processes = Sidekiq.redis { |conn|
         conn.sscan_each("processes").to_a
@@ -116,11 +126,13 @@ module Sidekiq
       @stats
     end
 
+    # @api private
     def fetch_stats!
       fetch_stats_fast!
       fetch_stats_slow!
     end
 
+    # @api private
     def reset(*stats)
       all = %w[failed processed]
       stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
@@ -202,9 +214,10 @@ module Sidekiq
   end
 
   ##
-  # Encapsulates a queue within Sidekiq.
+  # Represents a queue within Sidekiq.
   # Allows enumeration of all jobs within the queue
-  # and deletion of jobs.
+  # and deletion of jobs. NB: this queue data is real-time
+  # and is changing within Redis moment by moment.
   #
   #   queue = Sidekiq::Queue.new("mailer")
   #   queue.each do |job|
@@ -212,29 +225,34 @@ module Sidekiq
   #     job.args # => [1, 2, 3]
   #     job.delete if job.jid == 'abcdef1234567890'
   #   end
-  #
   class Queue
     include Enumerable
 
     ##
-    # Return all known queues within Redis.
+    # Fetch all known queues within Redis.
     #
+    # @return [Array<Sidekiq::Queue>]
     def self.all
       Sidekiq.redis { |c| c.sscan_each("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
     end
 
     attr_reader :name
 
+    # @param name [String] the name of the queue
     def initialize(name = "default")
       @name = name.to_s
       @rname = "queue:#{name}"
     end
 
+    # The current size of the queue within Redis.
+    # This value is real-time and can change between calls.
+    #
+    # @return [Integer] the size
     def size
       Sidekiq.redis { |con| con.llen(@rname) }
     end
 
-    # Sidekiq Pro overrides this
+    # @return [Boolean] if the queue is currently paused
     def paused?
       false
     end
@@ -243,7 +261,7 @@ module Sidekiq
     # Calculates this queue's latency, the difference in seconds since the oldest
     # job in the queue was enqueued.
     #
-    # @return Float
+    # @return [Float] in seconds
     def latency
       entry = Sidekiq.redis { |conn|
         conn.lrange(@rname, -1, -1)
@@ -279,12 +297,18 @@ module Sidekiq
     ##
     # Find the job with the given JID within this queue.
     #
-    # This is a slow, inefficient operation.  Do not use under
+    # This is a *slow, inefficient* operation.  Do not use under
     # normal conditions.
+    #
+    # @param jid [String] the job_id to look for
+    # @return [Sidekiq::JobRecord]
+    # @return [nil] if not found
     def find_job(jid)
       detect { |j| j.jid == jid }
     end
 
+    # delete all jobs within this queue
+    # @return [Boolean] true
     def clear
       Sidekiq.redis do |conn|
         conn.multi do |transaction|
@@ -292,25 +316,35 @@ module Sidekiq
           transaction.srem("queues", name)
         end
       end
+      true
     end
     alias_method :💣, :clear
 
-    def as_json(options = nil) # :nodoc:
+    # :nodoc:
+    # @api private
+    def as_json(options = nil)
       {name: name} # 5336
     end
   end
 
   ##
-  # Encapsulates a pending job within a Sidekiq queue or
-  # sorted set.
+  # Represents a pending job within a Sidekiq queue.
   #
   # The job should be considered immutable but may be
   # removed from the queue via JobRecord#delete.
-  #
   class JobRecord
+    # the parsed Hash of job data
+    # @!attribute [r] Item
     attr_reader :item
+    # the underlying String in Redis
+    # @!attribute [r] Value
     attr_reader :value
+    # the queue associated with this job
+    # @!attribute [r] Queue
+    attr_reader :queue
 
+    # :nodoc:
+    # @api private
     def initialize(item, queue_name = nil)
       @args = nil
       @value = item
@@ -318,6 +352,8 @@ module Sidekiq
       @queue = queue_name || @item["queue"]
     end
 
+    # :nodoc:
+    # @api private
     def parse(item)
       Sidekiq.load_json(item)
     rescue JSON::ParserError
@@ -329,6 +365,8 @@ module Sidekiq
       {}
     end
 
+    # This is the job class which Sidekiq will execute. If using ActiveJob,
+    # this class will be the ActiveJob adapter class rather than a specific job.
     def klass
       self["class"]
     end
@@ -416,15 +454,12 @@ module Sidekiq
       end
     end
 
-    attr_reader :queue
-
     def latency
       now = Time.now.to_f
       now - (@item["enqueued_at"] || @item["created_at"] || now)
     end
 
-    ##
-    # Remove this job from the queue.
+    # Remove this job from the queue
     def delete
       count = Sidekiq.redis { |conn|
         conn.lrem("queue:#{@queue}", 1, @value)
@@ -432,6 +467,7 @@ module Sidekiq
       count != 0
     end
 
+    # Access arbitrary attributes within the job hash
     def [](name)
       # nil will happen if the JSON fails to parse.
       # We don't guarantee Sidekiq will work with bad job JSON but we should
@@ -469,20 +505,28 @@ module Sidekiq
     end
   end
 
+  # Represents a job within a Redis sorted set where the score
+  # represents a timestamp associated with the job. This timestamp
+  # could be the scheduled time for it to run (e.g. scheduled set),
+  # or the expiration date after which the entry should be deleted (e.g. dead set).
   class SortedEntry < JobRecord
     attr_reader :score
     attr_reader :parent
 
+    # :nodoc:
+    # @api private
     def initialize(parent, score, item)
       super(item)
       @score = Float(score)
       @parent = parent
     end
 
+    # The timestamp associated with this entry
     def at
       Time.at(score).utc
     end
 
+    # remove this entry from the sorted set
     def delete
       if @value
         @parent.delete_by_value(@parent.name, @value)
@@ -491,12 +535,17 @@ module Sidekiq
       end
     end
 
+    # Change the scheduled time for this job.
+    #
+    # @param at [Time] the new timestamp for this job
     def reschedule(at)
       Sidekiq.redis do |conn|
         conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
       end
     end
 
+    # Enqueue this job from the scheduled or dead set so it will
+    # be executed at some point in the near future.
     def add_to_queue
       remove_job do |message|
         msg = Sidekiq.load_json(message)
@@ -504,6 +553,8 @@ module Sidekiq
       end
     end
 
+    # enqueue this job from the retry set so it will be executed
+    # at some point in the near future.
     def retry
       remove_job do |message|
         msg = Sidekiq.load_json(message)
@@ -512,8 +563,7 @@ module Sidekiq
       end
     end
 
-    ##
-    # Place job in the dead set
+    # Move this job from its current set into the Dead set.
     def kill
       remove_job do |message|
         DeadSet.new.kill(message)
@@ -561,20 +611,32 @@ module Sidekiq
     end
   end
 
+  # Base class for all sorted sets within Sidekiq.
   class SortedSet
     include Enumerable
 
+    # Redis key of the set
+    # @!attribute [r] Name
     attr_reader :name
 
+    # :nodoc:
+    # @api private
     def initialize(name)
       @name = name
       @_size = size
     end
 
+    # real-time size of the set, will change
     def size
       Sidekiq.redis { |c| c.zcard(name) }
     end
 
+    # Scan through each element of the sorted set, yielding each to the supplied block.
+    # Please see Redis's <a href="https://redis.io/commands/scan/">SCAN documentation</a> for implementation details.
+    #
+    # @param match [String] a snippet or regexp to filter matches.
+    # @param count [Integer] number of elements to retrieve at a time, default 100
+    # @yieldparam [Sidekiq::SortedEntry] each entry
     def scan(match, count = 100)
       return to_enum(:scan, match, count) unless block_given?
 
@@ -586,22 +648,32 @@ module Sidekiq
       end
     end
 
+    # @return [Boolean] always true
     def clear
       Sidekiq.redis do |conn|
         conn.unlink(name)
       end
+      true
     end
     alias_method :💣, :clear
 
-    def as_json(options = nil) # :nodoc:
+    # :nodoc:
+    # @api private
+    def as_json(options = nil)
       {name: name} # 5336
     end
   end
 
+  # Base class for all sorted sets which contain jobs, e.g. scheduled, retry and dead.
+  # Sidekiq Pro and Enterprise add additional sorted sets which do not contain job data,
+  # e.g. Batches.
   class JobSet < SortedSet
-    def schedule(timestamp, message)
+    # Add a job with the associated timestamp to this set.
+    # @param timestamp [Time] the score for the job
+    # @param job [Hash] the job data
+    def schedule(timestamp, job)
       Sidekiq.redis do |conn|
-        conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(message))
+        conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(job))
       end
     end
 
@@ -629,6 +701,10 @@ module Sidekiq
     ##
     # Fetch jobs that match a given time or Range. Job ID is an
     # optional second argument.
+    #
+    # @param score [Time,Range] a specific timestamp or range
+    # @param jid [String, optional] find a specific JID within the score
+    # @return [Array<SortedEntry>] any results found, can be empty
     def fetch(score, jid = nil)
       begin_score, end_score =
         if score.is_a?(Range)
@@ -650,7 +726,10 @@ module Sidekiq
 
     ##
     # Find the job with the given JID within this sorted set.
-    # This is a slower O(n) operation.  Do not use for app logic.
+    # *This is a slow O(n) operation*.  Do not use for app logic.
+    #
+    # @param jid [String] the job identifier
+    # @return [SortedEntry] the record or nil
     def find_job(jid)
       Sidekiq.redis do |conn|
         conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
@@ -662,6 +741,8 @@ module Sidekiq
       nil
     end
 
+    # :nodoc:
+    # @api private
     def delete_by_value(name, value)
       Sidekiq.redis do |conn|
         ret = conn.zrem(name, value)
@@ -670,6 +751,8 @@ module Sidekiq
       end
     end
 
+    # :nodoc:
+    # @api private
     def delete_by_jid(score, jid)
       Sidekiq.redis do |conn|
         elements = conn.zrangebyscore(name, score, score)
@@ -690,10 +773,10 @@ module Sidekiq
   end
 
   ##
-  # Allows enumeration of scheduled jobs within Sidekiq.
+  # The set of scheduled jobs within Sidekiq.
   # Based on this, you can search/filter for jobs.  Here's an
-  # example where I'm selecting all jobs of a certain type
-  # and deleting them from the schedule queue.
+  # example where I'm selecting jobs based on some complex logic
+  # and deleting them from the scheduled set.
   #
   #   r = Sidekiq::ScheduledSet.new
   #   r.select do |scheduled|
@@ -708,7 +791,7 @@ module Sidekiq
   end
 
   ##
-  # Allows enumeration of retries within Sidekiq.
+  # The set of retries within Sidekiq.
   # Based on this, you can search/filter for jobs.  Here's an
   # example where I'm selecting all jobs of a certain type
   # and deleting them from the retry queue.
@@ -724,23 +807,29 @@ module Sidekiq
       super "retry"
     end
 
+    # Enqueues all jobs pending within the retry set.
     def retry_all
       each(&:retry) while size > 0
     end
 
+    # Kills all jobs pending within the retry set.
     def kill_all
       each(&:kill) while size > 0
     end
   end
 
   ##
-  # Allows enumeration of dead jobs within Sidekiq.
+  # The set of dead jobs within Sidekiq. Dead jobs have failed all of
+  # their retries and are helding in this set pending some sort of manual
+  # fix. They will be removed after 6 months (dead_timeout) if not.
   #
   class DeadSet < JobSet
     def initialize
       super "dead"
     end
 
+    # Add the given job to the Dead set.
+    # @param message [String] the job data as JSON
     def kill(message, opts = {})
       now = Time.now.to_f
       Sidekiq.redis do |conn|
@@ -762,14 +851,19 @@ module Sidekiq
       true
     end
 
+    # Enqueue all dead jobs
     def retry_all
       each(&:retry) while size > 0
     end
 
+    # The maximum size of the Dead set. Older entries will be trimmed
+    # to stay within this limit. Default value is 10,000.
     def self.max_jobs
       Sidekiq[:dead_max_jobs]
     end
 
+    # The time limit for entries within the Dead set. Older entries will be thrown away.
+    # Default value is six months.
     def self.timeout
       Sidekiq[:dead_timeout_in_seconds]
     end
@@ -780,18 +874,23 @@ module Sidekiq
   # right now.  Each process sends a heartbeat to Redis every 5 seconds
   # so this set should be relatively accurate, barring network partitions.
   #
-  # Yields a Sidekiq::Process.
+  # @yieldparam [Sidekiq::Process]
   #
   class ProcessSet
     include Enumerable
 
+    # :nodoc:
+    # @api private
     def initialize(clean_plz = true)
       cleanup if clean_plz
     end
 
     # Cleans up dead processes recorded in Redis.
     # Returns the number of processes cleaned.
+    # :nodoc:
+    # @api private
     def cleanup
+      return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
       count = 0
       Sidekiq.redis do |conn|
         procs = conn.sscan_each("processes").to_a.sort
@@ -845,6 +944,7 @@ module Sidekiq
     # based on current heartbeat.  #each does that and ensures the set only
     # contains Sidekiq processes which have sent a heartbeat within the last
     # 60 seconds.
+    # @return [Integer] current number of registered Sidekiq processes
     def size
       Sidekiq.redis { |conn| conn.scard("processes") }
     end
@@ -852,10 +952,12 @@ module Sidekiq
     # Total number of threads available to execute jobs.
     # For Sidekiq Enterprise customers this number (in production) must be
     # less than or equal to your licensed concurrency.
+    # @return [Integer] the sum of process concurrency
     def total_concurrency
       sum { |x| x["concurrency"].to_i }
     end
 
+    # @return [Integer] total amount of RSS memory consumed by Sidekiq processes
     def total_rss_in_kb
       sum { |x| x["rss"].to_i }
     end
@@ -864,6 +966,8 @@ module Sidekiq
     # Returns the identity of the current cluster leader or "" if no leader.
     # This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
     # or Sidekiq Pro.
+    # @return [String] Identity of cluster leader
+    # @return [String] empty string if no leader
     def leader
       @leader ||= begin
         x = Sidekiq.redis { |c| c.get("dear-leader") }
@@ -890,6 +994,8 @@ module Sidekiq
   #   'identity' => <unique string identifying the process>,
   # }
   class Process
+    # :nodoc:
+    # @api private
     def initialize(hash)
       @attribs = hash
     end
@@ -914,18 +1020,31 @@ module Sidekiq
       self["queues"]
     end
 
+    # Signal this process to stop processing new jobs.
+    # It will continue to execute jobs it has already fetched.
+    # This method is *asynchronous* and it can take 5-10
+    # seconds for the process to quiet.
     def quiet!
       signal("TSTP")
     end
 
+    # Signal this process to shutdown.
+    # It will shutdown within its configured :timeout value, default 25 seconds.
+    # This method is *asynchronous* and it can take 5-10
+    # seconds for the process to start shutting down.
     def stop!
       signal("TERM")
     end
 
+    # Signal this process to log backtraces for all threads.
+    # Useful if you have a frozen or deadlocked process which is
+    # still sending a heartbeat.
+    # This method is *asynchronous* and it can take 5-10 seconds.
     def dump_threads
       signal("TTIN")
     end
 
+    # @return [Boolean] true if this process is quiet or shutting down
     def stopping?
       self["quiet"] == "true"
     end

data/lib/sidekiq/cli.rb

@@ -12,7 +12,7 @@ require "sidekiq"
 require "sidekiq/component"
 require "sidekiq/launcher"
 
-module Sidekiq
+module Sidekiq # :nodoc:
   class CLI
     include Sidekiq::Component
     include Singleton unless $TESTING
@@ -426,3 +426,4 @@ module Sidekiq
 end
 
 require "sidekiq/systemd"
+require "sidekiq/metrics/tracking" if ENV["SIDEKIQ_METRICS_BETA"]

data/lib/sidekiq/client.rb

@@ -220,7 +220,7 @@ module Sidekiq
 
     def atomic_push(conn, payloads)
       if payloads.first.key?("at")
-        conn.zadd("schedule", *payloads.map { |hash|
+        conn.zadd("schedule", payloads.flat_map { |hash|
           at = hash.delete("at").to_s
           [at, Sidekiq.dump_json(hash)]
         })

data/lib/sidekiq/component.rb

@@ -1,7 +1,7 @@
 module Sidekiq
   ##
   # Sidekiq::Component assumes a config instance is available at @config
-  module Component
+  module Component # :nodoc:
     attr_reader :config
 
     def watchdog(last_words)
@@ -47,6 +47,7 @@ module Sidekiq
     end
 
     def fire_event(event, options = {})
+      oneshot = options.fetch(:oneshot, true)
       reverse = options[:reverse]
       reraise = options[:reraise]
 
@@ -58,7 +59,7 @@ module Sidekiq
         handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
         raise ex if reraise
       end
-      arr.clear # once we've fired an event, we never fire it again
+      arr.clear if oneshot # once we've fired an event, we never fire it again
     end
   end
 end

data/lib/sidekiq/delay.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-module Sidekiq
+module Sidekiq # :nodoc:
   module Extensions
     def self.enable_delay!
       warn "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0", uplevel: 1

data/lib/sidekiq/fetch.rb

@@ -3,7 +3,7 @@
 require "sidekiq"
 require "sidekiq/component"
 
-module Sidekiq
+module Sidekiq # :nodoc:
   class BasicFetch
     include Sidekiq::Component
     # We want the fetch operation to timeout every few seconds so the thread