sidekiq 6.4.2 → 6.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70d899b76bdd764a2ec7e5f23b6d056494b58897daaf49db672daad3e0f60237
-  data.tar.gz: daae0430ecaf8eb7f172c70e628581c20a17e1a7cb1a58170b9082fc3dde1ea2
+  metadata.gz: f2c97836bf831cdce9bbab92fbd7b0d01d3522c24f5cced0e0822a53b906d0af
+  data.tar.gz: 85c7f9abe6844a5471519ca235c65e84275bd2f8d33e395ec3f85bada091c699
 SHA512:
-  metadata.gz: 930a0feb7ff47473eb995f8ba695468498afecb311c25296c93042ecfa00cd2bf48e19efd5ba775b4895c065705a344043e0c0e5d7b4115c2dceb1ec7cf341f9
-  data.tar.gz: 14ab645609e7c9f3fa954432097cd46d6e9e74e5bddc9ed1541697d13f3399add407ffb5af23c89c82651793c97e2c54b8fd4f845ca20287ce6bf033fd9caed2
+  metadata.gz: 06e0cad7074f7288c4ec66ee0f8e38e85c556390f61a23dc9dc025459a0da57ed0641ae89c450862c2101045d4220e5e11ad8b7a23f5316fbfe7fb12fc04ec67
+  data.tar.gz: 7362db4b038735acda91a86017053cc5fec9b70f1bbc8b0650abb592b60598c49a07fbecd3ec14b18a9f6b1cf6e3b2e0810b64e70e833072b64248a945f45646

data/Changes.md

@@ -2,6 +2,41 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
 
+6.5.2
+----------
+
+- [Job Metrics are under active development, help wanted!](https://github.com/mperham/sidekiq/wiki/Metrics#contributing) **BETA**
+- Add `Context` column on queue page which shows any CurrentAttributes [#5450]
+- `sidekiq_retry_in` may now return `:discard` or `:kill` to dynamically stop job retries [#5406]
+- Smarter sorting of processes in /busy Web UI [#5398]
+- Fix broken hamburger menu in mobile UI [#5428]
+- Require redis-rb 4.5.0. Note that Sidekiq will break if you use the
+  [`Redis.exists_returns_integer = false`](https://github.com/redis/redis-rb/blob/master/CHANGELOG.md#450) flag. [#5394]
+
+6.5.1
+----------
+
+- Fix `push_bulk` breakage [#5387]
+
+6.5.0
+---------
+
+- Substantial refactoring of Sidekiq server internals, part of a larger effort
+  to reduce Sidekiq's internal usage of global methods and data, see [docs/global_to_local.md](docs/global_to_local.md) and [docs/middleware.md](docs/middleware.md).
+- **Add beta support for the `redis-client` gem**. This will become the default Redis driver in Sidekiq 7.0. [#5298]
+  Read more: https://github.com/mperham/sidekiq/wiki/Using-redis-client
+- **Add beta support for DB transaction-aware client** [#5291]
+  Add this line to your initializer and any jobs created during a transaction
+  will only be pushed to Redis **after the transaction commits**. You will need to add the
+  `after_commit_everywhere` gem to your Gemfile.
+```ruby
+Sidekiq.transactional_push!
+```
+  This feature does not have a lot of production usage yet; please try it out and let us
+  know if you have any issues. It will be fully supported in Sidekiq 7.0 or removed if it
+  proves problematic.
+- Fix regression with middleware arguments [#5312]
+
 6.4.2
 ---------
 

data/bin/sidekiqload

@@ -11,6 +11,10 @@ Bundler.require(:default, :load_test)
 require_relative "../lib/sidekiq/cli"
 require_relative "../lib/sidekiq/launcher"
 
+if ENV["SIDEKIQ_REDIS_CLIENT"]
+  Sidekiq::RedisConnection.adapter = :redis_client
+end
+
 Sidekiq.configure_server do |config|
   config.options[:concurrency] = 10
   config.redis = {db: 13, port: 6380}
@@ -118,15 +122,23 @@ Monitoring = Thread.new do
   end
 end
 
+def with_latency(latency, &block)
+  Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
+  if latency > 0
+    Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
+  else
+    yield
+  end
+end
+
 begin
   # RubyProf::exclude_threads = [ Monitoring ]
   # RubyProf.start
   events = Sidekiq.options[:lifecycle_events][:startup]
   events.each(&:call)
   events.clear
-
-  Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
-  Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
+  
+  with_latency(Integer(ENV.fetch("LATENCY", "1"))) do
     launcher = Sidekiq::Launcher.new(Sidekiq.options)
     launcher.run
 

data/lib/sidekiq/api.rb

@@ -3,9 +3,17 @@
 require "sidekiq"
 
 require "zlib"
+require "set"
 require "base64"
+require "sidekiq/metrics/deploy"
+require "sidekiq/metrics/query"
 
 module Sidekiq
+  # Retrieve runtime statistics from Redis regarding
+  # this Sidekiq cluster.
+  #
+  #   stat = Sidekiq::Stats.new
+  #   stat.processed
   class Stats
     def initialize
       fetch_stats_fast!
@@ -52,6 +60,7 @@ module Sidekiq
     end
 
     # O(1) redis calls
+    # @api private
     def fetch_stats_fast!
       pipe1_res = Sidekiq.redis { |conn|
         conn.pipelined do |pipeline|
@@ -91,6 +100,7 @@ module Sidekiq
     end
 
     # O(number of processes + number of queues) redis calls
+    # @api private
     def fetch_stats_slow!
       processes = Sidekiq.redis { |conn|
         conn.sscan_each("processes").to_a
@@ -116,11 +126,13 @@ module Sidekiq
       @stats
     end
 
+    # @api private
     def fetch_stats!
       fetch_stats_fast!
       fetch_stats_slow!
     end
 
+    # @api private
     def reset(*stats)
       all = %w[failed processed]
       stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
@@ -191,7 +203,7 @@ module Sidekiq
               stat_hash[dates[idx]] = value ? value.to_i : 0
             end
           end
-        rescue Redis::CommandError
+        rescue RedisConnection.adapter::CommandError
           # mget will trigger a CROSSSLOT error when run against a Cluster
           # TODO Someone want to add Cluster support?
         end
@@ -202,9 +214,10 @@ module Sidekiq
   end
 
   ##
-  # Encapsulates a queue within Sidekiq.
+  # Represents a queue within Sidekiq.
   # Allows enumeration of all jobs within the queue
-  # and deletion of jobs.
+  # and deletion of jobs. NB: this queue data is real-time
+  # and is changing within Redis moment by moment.
   #
   #   queue = Sidekiq::Queue.new("mailer")
   #   queue.each do |job|
@@ -212,29 +225,34 @@ module Sidekiq
   #     job.args # => [1, 2, 3]
   #     job.delete if job.jid == 'abcdef1234567890'
   #   end
-  #
   class Queue
     include Enumerable
 
     ##
-    # Return all known queues within Redis.
+    # Fetch all known queues within Redis.
     #
+    # @return [Array<Sidekiq::Queue>]
     def self.all
       Sidekiq.redis { |c| c.sscan_each("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
     end
 
     attr_reader :name
 
+    # @param name [String] the name of the queue
     def initialize(name = "default")
       @name = name.to_s
       @rname = "queue:#{name}"
     end
 
+    # The current size of the queue within Redis.
+    # This value is real-time and can change between calls.
+    #
+    # @return [Integer] the size
     def size
       Sidekiq.redis { |con| con.llen(@rname) }
     end
 
-    # Sidekiq Pro overrides this
+    # @return [Boolean] if the queue is currently paused
     def paused?
       false
     end
@@ -243,7 +261,7 @@ module Sidekiq
     # Calculates this queue's latency, the difference in seconds since the oldest
     # job in the queue was enqueued.
     #
-    # @return Float
+    # @return [Float] in seconds
     def latency
       entry = Sidekiq.redis { |conn|
         conn.lrange(@rname, -1, -1)
@@ -279,12 +297,18 @@ module Sidekiq
     ##
     # Find the job with the given JID within this queue.
     #
-    # This is a slow, inefficient operation.  Do not use under
+    # This is a *slow, inefficient* operation.  Do not use under
     # normal conditions.
+    #
+    # @param jid [String] the job_id to look for
+    # @return [Sidekiq::JobRecord]
+    # @return [nil] if not found
     def find_job(jid)
       detect { |j| j.jid == jid }
     end
 
+    # delete all jobs within this queue
+    # @return [Boolean] true
     def clear
       Sidekiq.redis do |conn|
         conn.multi do |transaction|
@@ -292,21 +316,35 @@ module Sidekiq
           transaction.srem("queues", name)
         end
       end
+      true
     end
     alias_method :💣, :clear
+
+    # :nodoc:
+    # @api private
+    def as_json(options = nil)
+      {name: name} # 5336
+    end
   end
 
   ##
-  # Encapsulates a pending job within a Sidekiq queue or
-  # sorted set.
+  # Represents a pending job within a Sidekiq queue.
   #
   # The job should be considered immutable but may be
   # removed from the queue via JobRecord#delete.
-  #
   class JobRecord
+    # the parsed Hash of job data
+    # @!attribute [r] Item
     attr_reader :item
+    # the underlying String in Redis
+    # @!attribute [r] Value
     attr_reader :value
+    # the queue associated with this job
+    # @!attribute [r] Queue
+    attr_reader :queue
 
+    # :nodoc:
+    # @api private
     def initialize(item, queue_name = nil)
       @args = nil
       @value = item
@@ -314,6 +352,8 @@ module Sidekiq
       @queue = queue_name || @item["queue"]
     end
 
+    # :nodoc:
+    # @api private
     def parse(item)
       Sidekiq.load_json(item)
     rescue JSON::ParserError
@@ -325,6 +365,8 @@ module Sidekiq
       {}
     end
 
+    # This is the job class which Sidekiq will execute. If using ActiveJob,
+    # this class will be the ActiveJob adapter class rather than a specific job.
     def klass
       self["class"]
     end
@@ -412,15 +454,12 @@ module Sidekiq
       end
     end
 
-    attr_reader :queue
-
     def latency
       now = Time.now.to_f
       now - (@item["enqueued_at"] || @item["created_at"] || now)
     end
 
-    ##
-    # Remove this job from the queue.
+    # Remove this job from the queue
     def delete
       count = Sidekiq.redis { |conn|
         conn.lrem("queue:#{@queue}", 1, @value)
@@ -428,6 +467,7 @@ module Sidekiq
       count != 0
     end
 
+    # Access arbitrary attributes within the job hash
     def [](name)
       # nil will happen if the JSON fails to parse.
       # We don't guarantee Sidekiq will work with bad job JSON but we should
@@ -442,7 +482,8 @@ module Sidekiq
     rescue => ex
       # #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
       # memory yet so the YAML can't be loaded.
-      Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
+      # TODO is this still necessary? Zeitwerk reloader should handle?
+      Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.config[:environment] == "development"
       default
     end
 
@@ -464,20 +505,28 @@ module Sidekiq
     end
   end
 
+  # Represents a job within a Redis sorted set where the score
+  # represents a timestamp associated with the job. This timestamp
+  # could be the scheduled time for it to run (e.g. scheduled set),
+  # or the expiration date after which the entry should be deleted (e.g. dead set).
   class SortedEntry < JobRecord
     attr_reader :score
     attr_reader :parent
 
+    # :nodoc:
+    # @api private
     def initialize(parent, score, item)
       super(item)
-      @score = score
+      @score = Float(score)
       @parent = parent
     end
 
+    # The timestamp associated with this entry
     def at
       Time.at(score).utc
     end
 
+    # remove this entry from the sorted set
     def delete
       if @value
         @parent.delete_by_value(@parent.name, @value)
@@ -486,12 +535,17 @@ module Sidekiq
       end
     end
 
+    # Change the scheduled time for this job.
+    #
+    # @param at [Time] the new timestamp for this job
     def reschedule(at)
       Sidekiq.redis do |conn|
         conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
       end
     end
 
+    # Enqueue this job from the scheduled or dead set so it will
+    # be executed at some point in the near future.
     def add_to_queue
       remove_job do |message|
         msg = Sidekiq.load_json(message)
@@ -499,6 +553,8 @@ module Sidekiq
       end
     end
 
+    # enqueue this job from the retry set so it will be executed
+    # at some point in the near future.
     def retry
       remove_job do |message|
         msg = Sidekiq.load_json(message)
@@ -507,8 +563,7 @@ module Sidekiq
       end
     end
 
-    ##
-    # Place job in the dead set
+    # Move this job from its current set into the Dead set.
     def kill
       remove_job do |message|
         DeadSet.new.kill(message)
@@ -556,20 +611,32 @@ module Sidekiq
     end
   end
 
+  # Base class for all sorted sets within Sidekiq.
   class SortedSet
     include Enumerable
 
+    # Redis key of the set
+    # @!attribute [r] Name
     attr_reader :name
 
+    # :nodoc:
+    # @api private
     def initialize(name)
       @name = name
       @_size = size
     end
 
+    # real-time size of the set, will change
     def size
       Sidekiq.redis { |c| c.zcard(name) }
     end
 
+    # Scan through each element of the sorted set, yielding each to the supplied block.
+    # Please see Redis's <a href="https://redis.io/commands/scan/">SCAN documentation</a> for implementation details.
+    #
+    # @param match [String] a snippet or regexp to filter matches.
+    # @param count [Integer] number of elements to retrieve at a time, default 100
+    # @yieldparam [Sidekiq::SortedEntry] each entry
     def scan(match, count = 100)
       return to_enum(:scan, match, count) unless block_given?
 
@@ -581,18 +648,32 @@ module Sidekiq
       end
     end
 
+    # @return [Boolean] always true
     def clear
       Sidekiq.redis do |conn|
         conn.unlink(name)
       end
+      true
     end
     alias_method :💣, :clear
+
+    # :nodoc:
+    # @api private
+    def as_json(options = nil)
+      {name: name} # 5336
+    end
   end
 
+  # Base class for all sorted sets which contain jobs, e.g. scheduled, retry and dead.
+  # Sidekiq Pro and Enterprise add additional sorted sets which do not contain job data,
+  # e.g. Batches.
   class JobSet < SortedSet
-    def schedule(timestamp, message)
+    # Add a job with the associated timestamp to this set.
+    # @param timestamp [Time] the score for the job
+    # @param job [Hash] the job data
+    def schedule(timestamp, job)
       Sidekiq.redis do |conn|
-        conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(message))
+        conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(job))
       end
     end
 
@@ -606,7 +687,7 @@ module Sidekiq
         range_start = page * page_size + offset_size
         range_end = range_start + page_size - 1
         elements = Sidekiq.redis { |conn|
-          conn.zrange name, range_start, range_end, with_scores: true
+          conn.zrange name, range_start, range_end, withscores: true
         }
         break if elements.empty?
         page -= 1
@@ -620,6 +701,10 @@ module Sidekiq
     ##
     # Fetch jobs that match a given time or Range. Job ID is an
     # optional second argument.
+    #
+    # @param score [Time,Range] a specific timestamp or range
+    # @param jid [String, optional] find a specific JID within the score
+    # @return [Array<SortedEntry>] any results found, can be empty
     def fetch(score, jid = nil)
       begin_score, end_score =
         if score.is_a?(Range)
@@ -629,7 +714,7 @@ module Sidekiq
         end
 
       elements = Sidekiq.redis { |conn|
-        conn.zrangebyscore(name, begin_score, end_score, with_scores: true)
+        conn.zrangebyscore(name, begin_score, end_score, withscores: true)
       }
 
       elements.each_with_object([]) do |element, result|
@@ -641,7 +726,10 @@ module Sidekiq
 
     ##
     # Find the job with the given JID within this sorted set.
-    # This is a slower O(n) operation.  Do not use for app logic.
+    # *This is a slow O(n) operation*.  Do not use for app logic.
+    #
+    # @param jid [String] the job identifier
+    # @return [SortedEntry] the record or nil
     def find_job(jid)
       Sidekiq.redis do |conn|
         conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
@@ -653,6 +741,8 @@ module Sidekiq
       nil
     end
 
+    # :nodoc:
+    # @api private
     def delete_by_value(name, value)
       Sidekiq.redis do |conn|
         ret = conn.zrem(name, value)
@@ -661,6 +751,8 @@ module Sidekiq
       end
     end
 
+    # :nodoc:
+    # @api private
     def delete_by_jid(score, jid)
       Sidekiq.redis do |conn|
         elements = conn.zrangebyscore(name, score, score)
@@ -681,10 +773,10 @@ module Sidekiq
   end
 
   ##
-  # Allows enumeration of scheduled jobs within Sidekiq.
+  # The set of scheduled jobs within Sidekiq.
   # Based on this, you can search/filter for jobs.  Here's an
-  # example where I'm selecting all jobs of a certain type
-  # and deleting them from the schedule queue.
+  # example where I'm selecting jobs based on some complex logic
+  # and deleting them from the scheduled set.
   #
   #   r = Sidekiq::ScheduledSet.new
   #   r.select do |scheduled|
@@ -699,7 +791,7 @@ module Sidekiq
   end
 
   ##
-  # Allows enumeration of retries within Sidekiq.
+  # The set of retries within Sidekiq.
   # Based on this, you can search/filter for jobs.  Here's an
   # example where I'm selecting all jobs of a certain type
   # and deleting them from the retry queue.
@@ -715,23 +807,29 @@ module Sidekiq
       super "retry"
     end
 
+    # Enqueues all jobs pending within the retry set.
     def retry_all
       each(&:retry) while size > 0
     end
 
+    # Kills all jobs pending within the retry set.
     def kill_all
       each(&:kill) while size > 0
     end
   end
 
   ##
-  # Allows enumeration of dead jobs within Sidekiq.
+  # The set of dead jobs within Sidekiq. Dead jobs have failed all of
+  # their retries and are helding in this set pending some sort of manual
+  # fix. They will be removed after 6 months (dead_timeout) if not.
   #
   class DeadSet < JobSet
     def initialize
       super "dead"
     end
 
+    # Add the given job to the Dead set.
+    # @param message [String] the job data as JSON
     def kill(message, opts = {})
       now = Time.now.to_f
       Sidekiq.redis do |conn|
@@ -753,16 +851,21 @@ module Sidekiq
       true
     end
 
+    # Enqueue all dead jobs
     def retry_all
       each(&:retry) while size > 0
     end
 
+    # The maximum size of the Dead set. Older entries will be trimmed
+    # to stay within this limit. Default value is 10,000.
     def self.max_jobs
-      Sidekiq.options[:dead_max_jobs]
+      Sidekiq[:dead_max_jobs]
     end
 
+    # The time limit for entries within the Dead set. Older entries will be thrown away.
+    # Default value is six months.
     def self.timeout
-      Sidekiq.options[:dead_timeout_in_seconds]
+      Sidekiq[:dead_timeout_in_seconds]
     end
   end
 
@@ -771,18 +874,23 @@ module Sidekiq
   # right now.  Each process sends a heartbeat to Redis every 5 seconds
   # so this set should be relatively accurate, barring network partitions.
   #
-  # Yields a Sidekiq::Process.
+  # @yieldparam [Sidekiq::Process]
   #
   class ProcessSet
     include Enumerable
 
+    # :nodoc:
+    # @api private
     def initialize(clean_plz = true)
       cleanup if clean_plz
     end
 
     # Cleans up dead processes recorded in Redis.
     # Returns the number of processes cleaned.
+    # :nodoc:
+    # @api private
     def cleanup
+      return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
       count = 0
       Sidekiq.redis do |conn|
         procs = conn.sscan_each("processes").to_a.sort
@@ -836,6 +944,7 @@ module Sidekiq
     # based on current heartbeat.  #each does that and ensures the set only
     # contains Sidekiq processes which have sent a heartbeat within the last
     # 60 seconds.
+    # @return [Integer] current number of registered Sidekiq processes
     def size
       Sidekiq.redis { |conn| conn.scard("processes") }
     end
@@ -843,10 +952,12 @@ module Sidekiq
     # Total number of threads available to execute jobs.
     # For Sidekiq Enterprise customers this number (in production) must be
     # less than or equal to your licensed concurrency.
+    # @return [Integer] the sum of process concurrency
     def total_concurrency
       sum { |x| x["concurrency"].to_i }
     end
 
+    # @return [Integer] total amount of RSS memory consumed by Sidekiq processes
     def total_rss_in_kb
       sum { |x| x["rss"].to_i }
     end
@@ -855,6 +966,8 @@ module Sidekiq
     # Returns the identity of the current cluster leader or "" if no leader.
     # This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
     # or Sidekiq Pro.
+    # @return [String] Identity of cluster leader
+    # @return [String] empty string if no leader
     def leader
       @leader ||= begin
         x = Sidekiq.redis { |c| c.get("dear-leader") }
@@ -881,6 +994,8 @@ module Sidekiq
   #   'identity' => <unique string identifying the process>,
   # }
   class Process
+    # :nodoc:
+    # @api private
     def initialize(hash)
       @attribs = hash
     end
@@ -905,18 +1020,31 @@ module Sidekiq
       self["queues"]
     end
 
+    # Signal this process to stop processing new jobs.
+    # It will continue to execute jobs it has already fetched.
+    # This method is *asynchronous* and it can take 5-10
+    # seconds for the process to quiet.
     def quiet!
       signal("TSTP")
     end
 
+    # Signal this process to shutdown.
+    # It will shutdown within its configured :timeout value, default 25 seconds.
+    # This method is *asynchronous* and it can take 5-10
+    # seconds for the process to start shutting down.
     def stop!
       signal("TERM")
     end
 
+    # Signal this process to log backtraces for all threads.
+    # Useful if you have a frozen or deadlocked process which is
+    # still sending a heartbeat.
+    # This method is *asynchronous* and it can take 5-10 seconds.
     def dump_threads
       signal("TTIN")
     end
 
+    # @return [Boolean] true if this process is quiet or shutting down
     def stopping?
       self["quiet"] == "true"
     end