sidekiq 6.5.1 → 6.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1afbc6a1a0b14403e9148e746c08e0a2b24e634fca05288982c96719675607de
-  data.tar.gz: 3ff3f8df76b565f42030462eb8d09b673b89751dcd6a0b7c41a255789960d321
+  metadata.gz: f2c97836bf831cdce9bbab92fbd7b0d01d3522c24f5cced0e0822a53b906d0af
+  data.tar.gz: 85c7f9abe6844a5471519ca235c65e84275bd2f8d33e395ec3f85bada091c699
 SHA512:
-  metadata.gz: e8a68611735322d98cc517f1d03ef02394497f8eb505e0db496909cac1f6b7f0179f38ce1966546f717115395102c78fdad9ae0fd947360307b9288dcc22b369
-  data.tar.gz: 163e41dfb153a4e2ec50d407bde08cb8a395909807053f24478a73af1a6a9858b5aec92cbf618d4d066560a2b7bedaa2f88b48f823609e2d78f21320270cb97e
+  metadata.gz: 06e0cad7074f7288c4ec66ee0f8e38e85c556390f61a23dc9dc025459a0da57ed0641ae89c450862c2101045d4220e5e11ad8b7a23f5316fbfe7fb12fc04ec67
+  data.tar.gz: 7362db4b038735acda91a86017053cc5fec9b70f1bbc8b0650abb592b60598c49a07fbecd3ec14b18a9f6b1cf6e3b2e0810b64e70e833072b64248a945f45646

data/Changes.md

@@ -2,6 +2,17 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
 
+6.5.2
+----------
+
+- [Job Metrics are under active development, help wanted!](https://github.com/mperham/sidekiq/wiki/Metrics#contributing) **BETA**
+- Add `Context` column on queue page which shows any CurrentAttributes [#5450]
+- `sidekiq_retry_in` may now return `:discard` or `:kill` to dynamically stop job retries [#5406]
+- Smarter sorting of processes in /busy Web UI [#5398]
+- Fix broken hamburger menu in mobile UI [#5428]
+- Require redis-rb 4.5.0. Note that Sidekiq will break if you use the
+  [`Redis.exists_returns_integer = false`](https://github.com/redis/redis-rb/blob/master/CHANGELOG.md#450) flag. [#5394]
+
 6.5.1
 ----------
 

data/lib/sidekiq/api.rb

@@ -3,9 +3,17 @@
 require "sidekiq"
 
 require "zlib"
+require "set"
 require "base64"
+require "sidekiq/metrics/deploy"
+require "sidekiq/metrics/query"
 
 module Sidekiq
+  # Retrieve runtime statistics from Redis regarding
+  # this Sidekiq cluster.
+  #
+  #   stat = Sidekiq::Stats.new
+  #   stat.processed
   class Stats
     def initialize
       fetch_stats_fast!
@@ -52,6 +60,7 @@ module Sidekiq
     end
 
     # O(1) redis calls
+    # @api private
     def fetch_stats_fast!
       pipe1_res = Sidekiq.redis { |conn|
         conn.pipelined do |pipeline|
@@ -91,6 +100,7 @@ module Sidekiq
     end
 
     # O(number of processes + number of queues) redis calls
+    # @api private
     def fetch_stats_slow!
       processes = Sidekiq.redis { |conn|
         conn.sscan_each("processes").to_a
@@ -116,11 +126,13 @@ module Sidekiq
       @stats
     end
 
+    # @api private
     def fetch_stats!
       fetch_stats_fast!
       fetch_stats_slow!
     end
 
+    # @api private
     def reset(*stats)
       all = %w[failed processed]
       stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
@@ -202,9 +214,10 @@ module Sidekiq
   end
 
   ##
-  # Encapsulates a queue within Sidekiq.
+  # Represents a queue within Sidekiq.
   # Allows enumeration of all jobs within the queue
-  # and deletion of jobs.
+  # and deletion of jobs. NB: this queue data is real-time
+  # and is changing within Redis moment by moment.
   #
   #   queue = Sidekiq::Queue.new("mailer")
   #   queue.each do |job|
@@ -212,7 +225,6 @@ module Sidekiq
   #     job.args # => [1, 2, 3]
   #     job.delete if job.jid == 'abcdef1234567890'
   #   end
-  #
   class Queue
     include Enumerable
 
@@ -296,6 +308,7 @@ module Sidekiq
     end
 
     # delete all jobs within this queue
+    # @return [Boolean] true
     def clear
       Sidekiq.redis do |conn|
         conn.multi do |transaction|
@@ -303,34 +316,45 @@ module Sidekiq
           transaction.srem("queues", name)
         end
       end
+      true
     end
     alias_method :💣, :clear
 
-    def as_json(options = nil) # :nodoc:
+    # :nodoc:
+    # @api private
+    def as_json(options = nil)
       {name: name} # 5336
     end
   end
 
   ##
-  # Encapsulates a pending job within a Sidekiq queue or
-  # sorted set.
+  # Represents a pending job within a Sidekiq queue.
   #
   # The job should be considered immutable but may be
   # removed from the queue via JobRecord#delete.
-  #
   class JobRecord
+    # the parsed Hash of job data
+    # @!attribute [r] Item
     attr_reader :item
+    # the underlying String in Redis
+    # @!attribute [r] Value
     attr_reader :value
+    # the queue associated with this job
+    # @!attribute [r] Queue
     attr_reader :queue
 
-    def initialize(item, queue_name = nil) # :nodoc:
+    # :nodoc:
+    # @api private
+    def initialize(item, queue_name = nil)
       @args = nil
       @value = item
       @item = item.is_a?(Hash) ? item : parse(item)
       @queue = queue_name || @item["queue"]
     end
 
-    def parse(item) # :nodoc:
+    # :nodoc:
+    # @api private
+    def parse(item)
       Sidekiq.load_json(item)
     rescue JSON::ParserError
       # If the job payload in Redis is invalid JSON, we'll load
@@ -341,6 +365,8 @@ module Sidekiq
       {}
     end
 
+    # This is the job class which Sidekiq will execute. If using ActiveJob,
+    # this class will be the ActiveJob adapter class rather than a specific job.
     def klass
       self["class"]
     end
@@ -480,21 +506,27 @@ module Sidekiq
   end
 
   # Represents a job within a Redis sorted set where the score
-  # represents a timestamp for the job.
+  # represents a timestamp associated with the job. This timestamp
+  # could be the scheduled time for it to run (e.g. scheduled set),
+  # or the expiration date after which the entry should be deleted (e.g. dead set).
   class SortedEntry < JobRecord
     attr_reader :score
     attr_reader :parent
 
-    def initialize(parent, score, item) # :nodoc:
+    # :nodoc:
+    # @api private
+    def initialize(parent, score, item)
       super(item)
       @score = Float(score)
       @parent = parent
     end
 
+    # The timestamp associated with this entry
     def at
       Time.at(score).utc
     end
 
+    # remove this entry from the sorted set
     def delete
       if @value
         @parent.delete_by_value(@parent.name, @value)
@@ -505,7 +537,7 @@ module Sidekiq
 
     # Change the scheduled time for this job.
     #
-    # @param [Time] the new timestamp when this job will be enqueued.
+    # @param at [Time] the new timestamp for this job
     def reschedule(at)
       Sidekiq.redis do |conn|
         conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
@@ -579,20 +611,32 @@ module Sidekiq
     end
   end
 
+  # Base class for all sorted sets within Sidekiq.
   class SortedSet
     include Enumerable
 
+    # Redis key of the set
+    # @!attribute [r] Name
     attr_reader :name
 
+    # :nodoc:
+    # @api private
     def initialize(name)
       @name = name
       @_size = size
     end
 
+    # real-time size of the set, will change
     def size
       Sidekiq.redis { |c| c.zcard(name) }
     end
 
+    # Scan through each element of the sorted set, yielding each to the supplied block.
+    # Please see Redis's <a href="https://redis.io/commands/scan/">SCAN documentation</a> for implementation details.
+    #
+    # @param match [String] a snippet or regexp to filter matches.
+    # @param count [Integer] number of elements to retrieve at a time, default 100
+    # @yieldparam [Sidekiq::SortedEntry] each entry
     def scan(match, count = 100)
       return to_enum(:scan, match, count) unless block_given?
 
@@ -604,22 +648,32 @@ module Sidekiq
       end
     end
 
+    # @return [Boolean] always true
     def clear
       Sidekiq.redis do |conn|
         conn.unlink(name)
       end
+      true
     end
     alias_method :💣, :clear
 
-    def as_json(options = nil) # :nodoc:
+    # :nodoc:
+    # @api private
+    def as_json(options = nil)
       {name: name} # 5336
     end
   end
 
+  # Base class for all sorted sets which contain jobs, e.g. scheduled, retry and dead.
+  # Sidekiq Pro and Enterprise add additional sorted sets which do not contain job data,
+  # e.g. Batches.
   class JobSet < SortedSet
-    def schedule(timestamp, message)
+    # Add a job with the associated timestamp to this set.
+    # @param timestamp [Time] the score for the job
+    # @param job [Hash] the job data
+    def schedule(timestamp, job)
       Sidekiq.redis do |conn|
-        conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(message))
+        conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(job))
       end
     end
 
@@ -647,6 +701,10 @@ module Sidekiq
     ##
     # Fetch jobs that match a given time or Range. Job ID is an
     # optional second argument.
+    #
+    # @param score [Time,Range] a specific timestamp or range
+    # @param jid [String, optional] find a specific JID within the score
+    # @return [Array<SortedEntry>] any results found, can be empty
     def fetch(score, jid = nil)
       begin_score, end_score =
         if score.is_a?(Range)
@@ -668,7 +726,10 @@ module Sidekiq
 
     ##
     # Find the job with the given JID within this sorted set.
-    # This is a slower O(n) operation.  Do not use for app logic.
+    # *This is a slow O(n) operation*.  Do not use for app logic.
+    #
+    # @param jid [String] the job identifier
+    # @return [SortedEntry] the record or nil
     def find_job(jid)
       Sidekiq.redis do |conn|
         conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
@@ -680,6 +741,8 @@ module Sidekiq
       nil
     end
 
+    # :nodoc:
+    # @api private
     def delete_by_value(name, value)
       Sidekiq.redis do |conn|
         ret = conn.zrem(name, value)
@@ -688,6 +751,8 @@ module Sidekiq
       end
     end
 
+    # :nodoc:
+    # @api private
     def delete_by_jid(score, jid)
       Sidekiq.redis do |conn|
         elements = conn.zrangebyscore(name, score, score)
@@ -708,10 +773,10 @@ module Sidekiq
   end
 
   ##
-  # Allows enumeration of scheduled jobs within Sidekiq.
+  # The set of scheduled jobs within Sidekiq.
   # Based on this, you can search/filter for jobs.  Here's an
-  # example where I'm selecting all jobs of a certain type
-  # and deleting them from the schedule queue.
+  # example where I'm selecting jobs based on some complex logic
+  # and deleting them from the scheduled set.
   #
   #   r = Sidekiq::ScheduledSet.new
   #   r.select do |scheduled|
@@ -726,7 +791,7 @@ module Sidekiq
   end
 
   ##
-  # Allows enumeration of retries within Sidekiq.
+  # The set of retries within Sidekiq.
   # Based on this, you can search/filter for jobs.  Here's an
   # example where I'm selecting all jobs of a certain type
   # and deleting them from the retry queue.
@@ -742,23 +807,29 @@ module Sidekiq
       super "retry"
     end
 
+    # Enqueues all jobs pending within the retry set.
     def retry_all
       each(&:retry) while size > 0
     end
 
+    # Kills all jobs pending within the retry set.
     def kill_all
       each(&:kill) while size > 0
     end
   end
 
   ##
-  # Allows enumeration of dead jobs within Sidekiq.
+  # The set of dead jobs within Sidekiq. Dead jobs have failed all of
+  # their retries and are helding in this set pending some sort of manual
+  # fix. They will be removed after 6 months (dead_timeout) if not.
   #
   class DeadSet < JobSet
     def initialize
       super "dead"
     end
 
+    # Add the given job to the Dead set.
+    # @param message [String] the job data as JSON
     def kill(message, opts = {})
       now = Time.now.to_f
       Sidekiq.redis do |conn|
@@ -780,14 +851,19 @@ module Sidekiq
       true
     end
 
+    # Enqueue all dead jobs
     def retry_all
       each(&:retry) while size > 0
     end
 
+    # The maximum size of the Dead set. Older entries will be trimmed
+    # to stay within this limit. Default value is 10,000.
     def self.max_jobs
       Sidekiq[:dead_max_jobs]
     end
 
+    # The time limit for entries within the Dead set. Older entries will be thrown away.
+    # Default value is six months.
     def self.timeout
       Sidekiq[:dead_timeout_in_seconds]
     end
@@ -798,18 +874,23 @@ module Sidekiq
   # right now.  Each process sends a heartbeat to Redis every 5 seconds
   # so this set should be relatively accurate, barring network partitions.
   #
-  # Yields a Sidekiq::Process.
+  # @yieldparam [Sidekiq::Process]
   #
   class ProcessSet
     include Enumerable
 
+    # :nodoc:
+    # @api private
     def initialize(clean_plz = true)
       cleanup if clean_plz
     end
 
     # Cleans up dead processes recorded in Redis.
     # Returns the number of processes cleaned.
+    # :nodoc:
+    # @api private
     def cleanup
+      return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
       count = 0
       Sidekiq.redis do |conn|
         procs = conn.sscan_each("processes").to_a.sort
@@ -863,6 +944,7 @@ module Sidekiq
     # based on current heartbeat.  #each does that and ensures the set only
     # contains Sidekiq processes which have sent a heartbeat within the last
     # 60 seconds.
+    # @return [Integer] current number of registered Sidekiq processes
     def size
       Sidekiq.redis { |conn| conn.scard("processes") }
     end
@@ -870,10 +952,12 @@ module Sidekiq
     # Total number of threads available to execute jobs.
     # For Sidekiq Enterprise customers this number (in production) must be
     # less than or equal to your licensed concurrency.
+    # @return [Integer] the sum of process concurrency
     def total_concurrency
       sum { |x| x["concurrency"].to_i }
     end
 
+    # @return [Integer] total amount of RSS memory consumed by Sidekiq processes
     def total_rss_in_kb
       sum { |x| x["rss"].to_i }
     end
@@ -882,6 +966,8 @@ module Sidekiq
     # Returns the identity of the current cluster leader or "" if no leader.
     # This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
     # or Sidekiq Pro.
+    # @return [String] Identity of cluster leader
+    # @return [String] empty string if no leader
     def leader
       @leader ||= begin
         x = Sidekiq.redis { |c| c.get("dear-leader") }
@@ -908,6 +994,8 @@ module Sidekiq
   #   'identity' => <unique string identifying the process>,
   # }
   class Process
+    # :nodoc:
+    # @api private
     def initialize(hash)
       @attribs = hash
     end
@@ -932,18 +1020,31 @@ module Sidekiq
       self["queues"]
     end
 
+    # Signal this process to stop processing new jobs.
+    # It will continue to execute jobs it has already fetched.
+    # This method is *asynchronous* and it can take 5-10
+    # seconds for the process to quiet.
     def quiet!
       signal("TSTP")
     end
 
+    # Signal this process to shutdown.
+    # It will shutdown within its configured :timeout value, default 25 seconds.
+    # This method is *asynchronous* and it can take 5-10
+    # seconds for the process to start shutting down.
     def stop!
       signal("TERM")
     end
 
+    # Signal this process to log backtraces for all threads.
+    # Useful if you have a frozen or deadlocked process which is
+    # still sending a heartbeat.
+    # This method is *asynchronous* and it can take 5-10 seconds.
     def dump_threads
       signal("TTIN")
     end
 
+    # @return [Boolean] true if this process is quiet or shutting down
     def stopping?
       self["quiet"] == "true"
     end

data/lib/sidekiq/cli.rb

@@ -426,3 +426,4 @@ module Sidekiq # :nodoc:
 end
 
 require "sidekiq/systemd"
+require "sidekiq/metrics/tracking"

data/lib/sidekiq/component.rb

@@ -47,6 +47,7 @@ module Sidekiq
     end
 
     def fire_event(event, options = {})
+      oneshot = options.fetch(:oneshot, true)
       reverse = options[:reverse]
       reraise = options[:reraise]
 
@@ -58,7 +59,7 @@ module Sidekiq
         handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
         raise ex if reraise
       end
-      arr.clear # once we've fired an event, we never fire it again
+      arr.clear if oneshot # once we've fired an event, we never fire it again
     end
   end
 end

data/lib/sidekiq/job_retry.rb

@@ -91,7 +91,7 @@ module Sidekiq
 
       msg = Sidekiq.load_json(jobstr)
       if msg["retry"]
-        attempt_retry(nil, msg, queue, e)
+        process_retry(nil, msg, queue, e)
       else
         Sidekiq.death_handlers.each do |handler|
           handler.call(msg, e)
@@ -128,7 +128,7 @@ module Sidekiq
       end
 
       raise e unless msg["retry"]
-      attempt_retry(jobinst, msg, queue, e)
+      process_retry(jobinst, msg, queue, e)
       # We've handled this error associated with this job, don't
       # need to handle it at the global level
       raise Skip
@@ -139,7 +139,7 @@ module Sidekiq
     # Note that +jobinst+ can be nil here if an error is raised before we can
     # instantiate the job instance.  All access must be guarded and
     # best effort.
-    def attempt_retry(jobinst, msg, queue, exception)
+    def process_retry(jobinst, msg, queue, exception)
       max_retry_attempts = retry_attempts_from(msg["retry"], @max_retries)
 
       msg["queue"] = (msg["retry_queue"] || queue)
@@ -170,19 +170,50 @@ module Sidekiq
         msg["error_backtrace"] = compress_backtrace(lines)
       end
 
-      if count < max_retry_attempts
-        delay = delay_for(jobinst, count, exception)
-        # Logging here can break retries if the logging device raises ENOSPC #3979
-        # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
-        retry_at = Time.now.to_f + delay
-        payload = Sidekiq.dump_json(msg)
-        redis do |conn|
-          conn.zadd("retry", retry_at.to_s, payload)
-        end
+      # Goodbye dear message, you (re)tried your best I'm sure.
+      return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
+
+      strategy, delay = delay_for(jobinst, count, exception)
+      case strategy
+      when :discard
+        return # poof!
+      when :kill
+        return retries_exhausted(jobinst, msg, exception)
+      end
+
+      # Logging here can break retries if the logging device raises ENOSPC #3979
+      # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
+      jitter = rand(10) * (count + 1)
+      retry_at = Time.now.to_f + delay + jitter
+      payload = Sidekiq.dump_json(msg)
+      redis do |conn|
+        conn.zadd("retry", retry_at.to_s, payload)
+      end
+    end
+
+    # returns (strategy, seconds)
+    def delay_for(jobinst, count, exception)
+      rv = begin
+        # sidekiq_retry_in can return two different things:
+        # 1. When to retry next, as an integer of seconds
+        # 2. A symbol which re-routes the job elsewhere, e.g. :discard, :kill, :default
+        jobinst&.sidekiq_retry_in_block&.call(count, exception)
+      rescue Exception => e
+        handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
+        nil
+      end
+
+      delay = if Integer === rv && rv > 0
+        rv
+      elsif rv == :discard
+        return [:discard, nil] # do nothing, job goes poof
+      elsif rv == :kill
+        return [:kill, nil]
       else
-        # Goodbye dear message, you (re)tried your best I'm sure.
-        retries_exhausted(jobinst, msg, exception)
+        (count**4) + 15
       end
+
+      [:default, delay]
     end
 
     def retries_exhausted(jobinst, msg, exception)
@@ -216,22 +247,6 @@ module Sidekiq
       end
     end
 
-    def delay_for(jobinst, count, exception)
-      jitter = rand(10) * (count + 1)
-      if jobinst&.sidekiq_retry_in_block
-        custom_retry_in = retry_in(jobinst, count, exception).to_i
-        return custom_retry_in + jitter if custom_retry_in > 0
-      end
-      (count**4) + 15 + jitter
-    end
-
-    def retry_in(jobinst, count, exception)
-      jobinst.sidekiq_retry_in_block.call(count, exception)
-    rescue Exception => e
-      handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
-      nil
-    end
-
     def exception_caused_by_shutdown?(e, checked_causes = [])
       return false unless e.cause
 

data/lib/sidekiq/launcher.rb

@@ -79,6 +79,8 @@ module Sidekiq
     end
 
     def clear_heartbeat
+      flush_stats
+
       # Remove record from Redis since we are shutting down.
       # Note we don't stop the heartbeat thread; if the process
       # doesn't actually exit, it'll reappear in the Web UI.
@@ -98,7 +100,7 @@ module Sidekiq
       ❤
     end
 
-    def self.flush_stats
+    def flush_stats
       fails = Processor::FAILURE.reset
       procd = Processor::PROCESSED.reset
       return if fails + procd == 0
@@ -122,7 +124,6 @@ module Sidekiq
         Sidekiq.logger.warn("Unable to flush stats: #{ex}")
       end
     end
-    at_exit(&method(:flush_stats))
 
     def ❤
       key = identity
@@ -179,6 +180,7 @@ module Sidekiq
 
         # first heartbeat or recovering from an outage and need to reestablish our heartbeat
         fire_event(:heartbeat) unless exists
+        fire_event(:beat, oneshot: false)
 
         return unless msg
 

data/lib/sidekiq/metrics/deploy.rb

@@ -0,0 +1,47 @@
+require "sidekiq"
+require "date"
+
+# This file is designed to be required within the user's
+# deployment script; it should need a bare minimum of dependencies.
+#
+#   require "sidekiq/metrics/deploy"
+#   gitdesc = `git log -1 --format="%h %s"`.strip
+#   d = Sidekiq::Metrics::Deploy.new
+#   d.mark(label: gitdesc)
+#
+# Note that you cannot mark more than once per minute. This is a feature, not a bug.
+module Sidekiq
+  module Metrics
+    class Deploy
+      MARK_TTL = 90 * 24 * 60 * 60 # 90 days
+
+      def initialize(pool = Sidekiq.redis_pool)
+        @pool = pool
+      end
+
+      def mark(at: Time.now, label: "")
+        # we need to round the timestamp so that we gracefully
+        # handle an excepted common error in marking deploys:
+        # having every process mark its deploy, leading
+        # to N marks for each deploy. Instead we round the time
+        # to the minute so that multple marks within that minute
+        # will all naturally rollup into one mark per minute.
+        whence = at.utc
+        floor = Time.utc(whence.year, whence.month, whence.mday, whence.hour, whence.min, 0)
+        datecode = floor.strftime("%Y%m%d")
+        key = "#{datecode}-marks"
+        @pool.with do |c|
+          c.pipelined do |pipe|
+            pipe.hsetnx(key, floor.rfc3339, label)
+            pipe.expire(key, MARK_TTL)
+          end
+        end
+      end
+
+      def fetch(date = Time.now.utc.to_date)
+        datecode = date.strftime("%Y%m%d")
+        @pool.with { |c| c.hgetall("#{datecode}-marks") }
+      end
+    end
+  end
+end