sidekiq 6.5.1 → 6.5.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1afbc6a1a0b14403e9148e746c08e0a2b24e634fca05288982c96719675607de
-  data.tar.gz: 3ff3f8df76b565f42030462eb8d09b673b89751dcd6a0b7c41a255789960d321
+  metadata.gz: 55c6f9a8c0f2810bcbe7744c95204893d73f534666f6091c74dcb5199e7a4a28
+  data.tar.gz: c42690ef0d1876c94eb51fb68319275d11f10169826180db921b34d6334a1a54
 SHA512:
-  metadata.gz: e8a68611735322d98cc517f1d03ef02394497f8eb505e0db496909cac1f6b7f0179f38ce1966546f717115395102c78fdad9ae0fd947360307b9288dcc22b369
-  data.tar.gz: 163e41dfb153a4e2ec50d407bde08cb8a395909807053f24478a73af1a6a9858b5aec92cbf618d4d066560a2b7bedaa2f88b48f823609e2d78f21320270cb97e
+  metadata.gz: 461b559e18cbdf8fe6ba20ab9fa38d08fd3b7b8d14dce7a7f9369d678e92bd25c07734bc14b0167f83589c0f03501897195b3fdf9cfd5de5a60f039bf7436f98
+  data.tar.gz: ce0490b438a22a71c37e5a65193a7728e3297b437730d30a3653807ff1e89476db1f8c62d8de50c0c8cebbbce2ba6fb65f110855e071c62746fe8697a0f636e1

data/Changes.md

@@ -2,6 +2,71 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
 
+6.5.11
+----------
+
+- Fix for Rails 7.1 [#6067]
+
+6.5.10
+----------
+
+- Web UI DoS vector [#6045] CVE-2023-26141
+- Fix broadcast logger with Rails 7.1 [#6054]
+
+6.5.9
+----------
+
+- Ensure Sidekiq.options[:environment] == RAILS_ENV [#5932]
+
+6.5.8
+----------
+
+- Fail if using a bad version of scout_apm [#5616]
+- Add pagination to Busy page [#5556]
+- Speed up WorkSet#each [#5559]
+- Adjust CurrentAttributes to work with the String class name so we aren't referencing
+the Class within a Rails initializer [#5536]
+
+6.5.7
+----------
+
+- Updates for JA and ZH locales
+- Further optimizations for scheduled polling [#5513]
+
+6.5.6
+----------
+
+- Fix deprecation warnings with redis-rb 4.8.0 [#5484]
+- Lock redis-rb to < 5.0 as we are moving to redis-client in Sidekiq 7.0
+
+6.5.5
+----------
+
+- Fix require issue with job_retry.rb [#5462]
+- Improve Sidekiq::Web compatibility with Rack 3.x
+
+6.5.4
+----------
+
+- Fix invalid code on Ruby 2.5 [#5460]
+- Fix further metrics dependency issues [#5457]
+
+6.5.3
+----------
+
+- Don't require metrics code without explicit opt-in [#5456]
+
+6.5.2
+----------
+
+- [Job Metrics are under active development, help wanted!](https://github.com/mperham/sidekiq/wiki/Metrics#contributing) **BETA**
+- Add `Context` column on queue page which shows any CurrentAttributes [#5450]
+- `sidekiq_retry_in` may now return `:discard` or `:kill` to dynamically stop job retries [#5406]
+- Smarter sorting of processes in /busy Web UI [#5398]
+- Fix broken hamburger menu in mobile UI [#5428]
+- Require redis-rb 4.5.0. Note that Sidekiq will break if you use the
+  [`Redis.exists_returns_integer = false`](https://github.com/redis/redis-rb/blob/master/CHANGELOG.md#450) flag. [#5394]
+
 6.5.1
 ----------
 

data/bin/sidekiqload

@@ -89,7 +89,7 @@ def Process.rss
   `ps -o rss= -p #{Process.pid}`.chomp.to_i
 end
 
-iter = 50
+iter = 10
 count = 10_000
 
 iter.times do
@@ -139,7 +139,7 @@ begin
   events.clear
   
   with_latency(Integer(ENV.fetch("LATENCY", "1"))) do
-    launcher = Sidekiq::Launcher.new(Sidekiq.options)
+    launcher = Sidekiq::Launcher.new(Sidekiq)
     launcher.run
 
     while readable_io = IO.select([self_read])

data/lib/sidekiq/api.rb

@@ -3,9 +3,31 @@
 require "sidekiq"
 
 require "zlib"
+require "set"
 require "base64"
 
+if ENV["SIDEKIQ_METRICS_BETA"]
+  require "sidekiq/metrics/deploy"
+  require "sidekiq/metrics/query"
+end
+
+#
+# Sidekiq's Data API provides a Ruby object model on top
+# of Sidekiq's runtime data in Redis. This API should never
+# be used within application code for business logic.
+#
+# The Sidekiq server process never uses this API: all data
+# manipulation is done directly for performance reasons to
+# ensure we are using Redis as efficiently as possible at
+# every callsite.
+#
+
 module Sidekiq
+  # Retrieve runtime statistics from Redis regarding
+  # this Sidekiq cluster.
+  #
+  #   stat = Sidekiq::Stats.new
+  #   stat.processed
   class Stats
     def initialize
       fetch_stats_fast!
@@ -52,6 +74,7 @@ module Sidekiq
     end
 
     # O(1) redis calls
+    # @api private
     def fetch_stats_fast!
       pipe1_res = Sidekiq.redis { |conn|
         conn.pipelined do |pipeline|
@@ -91,6 +114,7 @@ module Sidekiq
     end
 
     # O(number of processes + number of queues) redis calls
+    # @api private
     def fetch_stats_slow!
       processes = Sidekiq.redis { |conn|
         conn.sscan_each("processes").to_a
@@ -116,11 +140,13 @@ module Sidekiq
       @stats
     end
 
+    # @api private
     def fetch_stats!
       fetch_stats_fast!
       fetch_stats_slow!
     end
 
+    # @api private
     def reset(*stats)
       all = %w[failed processed]
       stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
@@ -202,9 +228,10 @@ module Sidekiq
   end
 
   ##
-  # Encapsulates a queue within Sidekiq.
+  # Represents a queue within Sidekiq.
   # Allows enumeration of all jobs within the queue
-  # and deletion of jobs.
+  # and deletion of jobs. NB: this queue data is real-time
+  # and is changing within Redis moment by moment.
   #
   #   queue = Sidekiq::Queue.new("mailer")
   #   queue.each do |job|
@@ -212,7 +239,6 @@ module Sidekiq
   #     job.args # => [1, 2, 3]
   #     job.delete if job.jid == 'abcdef1234567890'
   #   end
-  #
   class Queue
     include Enumerable
 
@@ -296,41 +322,53 @@ module Sidekiq
     end
 
     # delete all jobs within this queue
+    # @return [Boolean] true
     def clear
       Sidekiq.redis do |conn|
         conn.multi do |transaction|
           transaction.unlink(@rname)
-          transaction.srem("queues", name)
+          transaction.srem("queues", [name])
         end
       end
+      true
     end
     alias_method :💣, :clear
 
-    def as_json(options = nil) # :nodoc:
+    # :nodoc:
+    # @api private
+    def as_json(options = nil)
       {name: name} # 5336
     end
   end
 
   ##
-  # Encapsulates a pending job within a Sidekiq queue or
-  # sorted set.
+  # Represents a pending job within a Sidekiq queue.
   #
   # The job should be considered immutable but may be
   # removed from the queue via JobRecord#delete.
-  #
   class JobRecord
+    # the parsed Hash of job data
+    # @!attribute [r] Item
     attr_reader :item
+    # the underlying String in Redis
+    # @!attribute [r] Value
     attr_reader :value
+    # the queue associated with this job
+    # @!attribute [r] Queue
     attr_reader :queue
 
-    def initialize(item, queue_name = nil) # :nodoc:
+    # :nodoc:
+    # @api private
+    def initialize(item, queue_name = nil)
       @args = nil
       @value = item
       @item = item.is_a?(Hash) ? item : parse(item)
       @queue = queue_name || @item["queue"]
     end
 
-    def parse(item) # :nodoc:
+    # :nodoc:
+    # @api private
+    def parse(item)
       Sidekiq.load_json(item)
     rescue JSON::ParserError
       # If the job payload in Redis is invalid JSON, we'll load
@@ -341,6 +379,8 @@ module Sidekiq
       {}
     end
 
+    # This is the job class which Sidekiq will execute. If using ActiveJob,
+    # this class will be the ActiveJob adapter class rather than a specific job.
     def klass
       self["class"]
     end
@@ -457,7 +497,7 @@ module Sidekiq
       # #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
       # memory yet so the YAML can't be loaded.
       # TODO is this still necessary? Zeitwerk reloader should handle?
-      Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.config[:environment] == "development"
+      Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
       default
     end
 
@@ -480,21 +520,27 @@ module Sidekiq
   end
 
   # Represents a job within a Redis sorted set where the score
-  # represents a timestamp for the job.
+  # represents a timestamp associated with the job. This timestamp
+  # could be the scheduled time for it to run (e.g. scheduled set),
+  # or the expiration date after which the entry should be deleted (e.g. dead set).
   class SortedEntry < JobRecord
     attr_reader :score
     attr_reader :parent
 
-    def initialize(parent, score, item) # :nodoc:
+    # :nodoc:
+    # @api private
+    def initialize(parent, score, item)
       super(item)
       @score = Float(score)
       @parent = parent
     end
 
+    # The timestamp associated with this entry
     def at
       Time.at(score).utc
     end
 
+    # remove this entry from the sorted set
     def delete
       if @value
         @parent.delete_by_value(@parent.name, @value)
@@ -505,7 +551,7 @@ module Sidekiq
 
     # Change the scheduled time for this job.
     #
-    # @param [Time] the new timestamp when this job will be enqueued.
+    # @param at [Time] the new timestamp for this job
     def reschedule(at)
       Sidekiq.redis do |conn|
         conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
@@ -579,20 +625,32 @@ module Sidekiq
     end
   end
 
+  # Base class for all sorted sets within Sidekiq.
   class SortedSet
     include Enumerable
 
+    # Redis key of the set
+    # @!attribute [r] Name
     attr_reader :name
 
+    # :nodoc:
+    # @api private
     def initialize(name)
       @name = name
       @_size = size
     end
 
+    # real-time size of the set, will change
     def size
       Sidekiq.redis { |c| c.zcard(name) }
     end
 
+    # Scan through each element of the sorted set, yielding each to the supplied block.
+    # Please see Redis's <a href="https://redis.io/commands/scan/">SCAN documentation</a> for implementation details.
+    #
+    # @param match [String] a snippet or regexp to filter matches.
+    # @param count [Integer] number of elements to retrieve at a time, default 100
+    # @yieldparam [Sidekiq::SortedEntry] each entry
     def scan(match, count = 100)
       return to_enum(:scan, match, count) unless block_given?
 
@@ -604,22 +662,32 @@ module Sidekiq
       end
     end
 
+    # @return [Boolean] always true
     def clear
       Sidekiq.redis do |conn|
         conn.unlink(name)
       end
+      true
     end
     alias_method :💣, :clear
 
-    def as_json(options = nil) # :nodoc:
+    # :nodoc:
+    # @api private
+    def as_json(options = nil)
       {name: name} # 5336
     end
   end
 
+  # Base class for all sorted sets which contain jobs, e.g. scheduled, retry and dead.
+  # Sidekiq Pro and Enterprise add additional sorted sets which do not contain job data,
+  # e.g. Batches.
   class JobSet < SortedSet
-    def schedule(timestamp, message)
+    # Add a job with the associated timestamp to this set.
+    # @param timestamp [Time] the score for the job
+    # @param job [Hash] the job data
+    def schedule(timestamp, job)
       Sidekiq.redis do |conn|
-        conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(message))
+        conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(job))
       end
     end
 
@@ -647,6 +715,10 @@ module Sidekiq
     ##
     # Fetch jobs that match a given time or Range. Job ID is an
     # optional second argument.
+    #
+    # @param score [Time,Range] a specific timestamp or range
+    # @param jid [String, optional] find a specific JID within the score
+    # @return [Array<SortedEntry>] any results found, can be empty
     def fetch(score, jid = nil)
       begin_score, end_score =
         if score.is_a?(Range)
@@ -668,7 +740,10 @@ module Sidekiq
 
     ##
     # Find the job with the given JID within this sorted set.
-    # This is a slower O(n) operation.  Do not use for app logic.
+    # *This is a slow O(n) operation*.  Do not use for app logic.
+    #
+    # @param jid [String] the job identifier
+    # @return [SortedEntry] the record or nil
     def find_job(jid)
       Sidekiq.redis do |conn|
         conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
@@ -680,6 +755,8 @@ module Sidekiq
       nil
     end
 
+    # :nodoc:
+    # @api private
     def delete_by_value(name, value)
       Sidekiq.redis do |conn|
         ret = conn.zrem(name, value)
@@ -688,6 +765,8 @@ module Sidekiq
       end
     end
 
+    # :nodoc:
+    # @api private
     def delete_by_jid(score, jid)
       Sidekiq.redis do |conn|
         elements = conn.zrangebyscore(name, score, score)
@@ -708,10 +787,10 @@ module Sidekiq
   end
 
   ##
-  # Allows enumeration of scheduled jobs within Sidekiq.
+  # The set of scheduled jobs within Sidekiq.
   # Based on this, you can search/filter for jobs.  Here's an
-  # example where I'm selecting all jobs of a certain type
-  # and deleting them from the schedule queue.
+  # example where I'm selecting jobs based on some complex logic
+  # and deleting them from the scheduled set.
   #
   #   r = Sidekiq::ScheduledSet.new
   #   r.select do |scheduled|
@@ -726,7 +805,7 @@ module Sidekiq
   end
 
   ##
-  # Allows enumeration of retries within Sidekiq.
+  # The set of retries within Sidekiq.
   # Based on this, you can search/filter for jobs.  Here's an
   # example where I'm selecting all jobs of a certain type
   # and deleting them from the retry queue.
@@ -742,23 +821,29 @@ module Sidekiq
       super "retry"
     end
 
+    # Enqueues all jobs pending within the retry set.
     def retry_all
       each(&:retry) while size > 0
     end
 
+    # Kills all jobs pending within the retry set.
     def kill_all
       each(&:kill) while size > 0
     end
   end
 
   ##
-  # Allows enumeration of dead jobs within Sidekiq.
+  # The set of dead jobs within Sidekiq. Dead jobs have failed all of
+  # their retries and are helding in this set pending some sort of manual
+  # fix. They will be removed after 6 months (dead_timeout) if not.
   #
   class DeadSet < JobSet
     def initialize
       super "dead"
     end
 
+    # Add the given job to the Dead set.
+    # @param message [String] the job data as JSON
     def kill(message, opts = {})
       now = Time.now.to_f
       Sidekiq.redis do |conn|
@@ -780,14 +865,19 @@ module Sidekiq
       true
     end
 
+    # Enqueue all dead jobs
     def retry_all
       each(&:retry) while size > 0
     end
 
+    # The maximum size of the Dead set. Older entries will be trimmed
+    # to stay within this limit. Default value is 10,000.
     def self.max_jobs
       Sidekiq[:dead_max_jobs]
     end
 
+    # The time limit for entries within the Dead set. Older entries will be thrown away.
+    # Default value is six months.
     def self.timeout
       Sidekiq[:dead_timeout_in_seconds]
     end
@@ -798,21 +888,28 @@ module Sidekiq
   # right now.  Each process sends a heartbeat to Redis every 5 seconds
   # so this set should be relatively accurate, barring network partitions.
   #
-  # Yields a Sidekiq::Process.
+  # @yieldparam [Sidekiq::Process]
   #
   class ProcessSet
     include Enumerable
 
+    # :nodoc:
+    # @api private
     def initialize(clean_plz = true)
       cleanup if clean_plz
     end
 
     # Cleans up dead processes recorded in Redis.
     # Returns the number of processes cleaned.
+    # :nodoc:
+    # @api private
     def cleanup
+      # dont run cleanup more than once per minute
+      return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
+
       count = 0
       Sidekiq.redis do |conn|
-        procs = conn.sscan_each("processes").to_a.sort
+        procs = conn.sscan_each("processes").to_a
         heartbeats = conn.pipelined { |pipeline|
           procs.each do |key|
             pipeline.hget(key, "info")
@@ -863,6 +960,7 @@ module Sidekiq
     # based on current heartbeat.  #each does that and ensures the set only
     # contains Sidekiq processes which have sent a heartbeat within the last
     # 60 seconds.
+    # @return [Integer] current number of registered Sidekiq processes
     def size
       Sidekiq.redis { |conn| conn.scard("processes") }
     end
@@ -870,10 +968,12 @@ module Sidekiq
     # Total number of threads available to execute jobs.
     # For Sidekiq Enterprise customers this number (in production) must be
     # less than or equal to your licensed concurrency.
+    # @return [Integer] the sum of process concurrency
     def total_concurrency
       sum { |x| x["concurrency"].to_i }
     end
 
+    # @return [Integer] total amount of RSS memory consumed by Sidekiq processes
     def total_rss_in_kb
       sum { |x| x["rss"].to_i }
     end
@@ -882,6 +982,8 @@ module Sidekiq
     # Returns the identity of the current cluster leader or "" if no leader.
     # This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
     # or Sidekiq Pro.
+    # @return [String] Identity of cluster leader
+    # @return [String] empty string if no leader
     def leader
       @leader ||= begin
         x = Sidekiq.redis { |c| c.get("dear-leader") }
@@ -908,6 +1010,8 @@ module Sidekiq
   #   'identity' => <unique string identifying the process>,
   # }
   class Process
+    # :nodoc:
+    # @api private
     def initialize(hash)
       @attribs = hash
     end
@@ -932,18 +1036,31 @@ module Sidekiq
       self["queues"]
     end
 
+    # Signal this process to stop processing new jobs.
+    # It will continue to execute jobs it has already fetched.
+    # This method is *asynchronous* and it can take 5-10
+    # seconds for the process to quiet.
     def quiet!
       signal("TSTP")
     end
 
+    # Signal this process to shutdown.
+    # It will shutdown within its configured :timeout value, default 25 seconds.
+    # This method is *asynchronous* and it can take 5-10
+    # seconds for the process to start shutting down.
     def stop!
       signal("TERM")
     end
 
+    # Signal this process to log backtraces for all threads.
+    # Useful if you have a frozen or deadlocked process which is
+    # still sending a heartbeat.
+    # This method is *asynchronous* and it can take 5-10 seconds.
     def dump_threads
       signal("TTIN")
     end
 
+    # @return [Boolean] true if this process is quiet or shutting down
     def stopping?
       self["quiet"] == "true"
     end
@@ -986,24 +1103,31 @@ module Sidekiq
 
     def each(&block)
       results = []
+      procs = nil
+      all_works = nil
+
       Sidekiq.redis do |conn|
-        procs = conn.sscan_each("processes").to_a
-        procs.sort.each do |key|
-          valid, workers = conn.pipelined { |pipeline|
-            pipeline.exists?(key)
+        procs = conn.sscan_each("processes").to_a.sort
+
+        all_works = conn.pipelined do |pipeline|
+          procs.each do |key|
             pipeline.hgetall("#{key}:work")
-          }
-          next unless valid
-          workers.each_pair do |tid, json|
-            hsh = Sidekiq.load_json(json)
-            p = hsh["payload"]
-            # avoid breaking API, this is a side effect of the JSON optimization in #4316
-            hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
-            results << [key, tid, hsh]
           end
         end
       end
 
+      procs.zip(all_works).each do |key, workers|
+        workers.each_pair do |tid, json|
+          next if json.empty?
+
+          hsh = Sidekiq.load_json(json)
+          p = hsh["payload"]
+          # avoid breaking API, this is a side effect of the JSON optimization in #4316
+          hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
+          results << [key, tid, hsh]
+        end
+      end
+
       results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
     end
 

data/lib/sidekiq/cli.rb

@@ -12,6 +12,17 @@ require "sidekiq"
 require "sidekiq/component"
 require "sidekiq/launcher"
 
+# module ScoutApm
+# VERSION = "5.3.1"
+# end
+fail <<~EOM if defined?(ScoutApm::VERSION) && ScoutApm::VERSION < "5.2.0"
+  
+  
+  scout_apm v#{ScoutApm::VERSION} is unsafe with Sidekiq 6.5. Please run `bundle up scout_apm` to upgrade to 5.2.0 or greater.
+  
+  
+EOM
+
 module Sidekiq # :nodoc:
   class CLI
     include Sidekiq::Component
@@ -214,6 +225,7 @@ module Sidekiq # :nodoc:
       # Both Sinatra 2.0+ and Sidekiq support this term.
       # RAILS_ENV and RACK_ENV are there for legacy support.
       @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      config[:environment] = @environment
     end
 
     def symbolize_keys_deep!(hash)
@@ -426,3 +438,4 @@ module Sidekiq # :nodoc:
 end
 
 require "sidekiq/systemd"
+require "sidekiq/metrics/tracking" if ENV["SIDEKIQ_METRICS_BETA"]

data/lib/sidekiq/client.rb

@@ -176,7 +176,7 @@ module Sidekiq
       def enqueue_to_in(queue, interval, klass, *args)
         int = interval.to_f
         now = Time.now.to_f
-        ts = (int < 1_000_000_000 ? now + int : int)
+        ts = ((int < 1_000_000_000) ? now + int : int)
 
         item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
         item.delete("at") if ts <= now
@@ -231,7 +231,7 @@ module Sidekiq
           entry["enqueued_at"] = now
           Sidekiq.dump_json(entry)
         }
-        conn.sadd("queues", queue)
+        conn.sadd("queues", [queue])
         conn.lpush("queue:#{queue}", to_push)
       end
     end

data/lib/sidekiq/component.rb

@@ -47,6 +47,7 @@ module Sidekiq
     end
 
     def fire_event(event, options = {})
+      oneshot = options.fetch(:oneshot, true)
       reverse = options[:reverse]
       reraise = options[:reraise]
 
@@ -58,7 +59,7 @@ module Sidekiq
         handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
         raise ex if reraise
       end
-      arr.clear # once we've fired an event, we never fire it again
+      arr.clear if oneshot # once we've fired an event, we never fire it again
     end
   end
 end