sidekiq 6.2.1 → 6.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 298711914bcb8534a9599c47b00b7410467ce324619ee70e7050d15c42f4c329
-  data.tar.gz: '007900de7a1558633520c61870a58eff341e9c11009441dbabe0fbc177e4ed99'
+  metadata.gz: 1b0da541124d097e05936cc860a93009ba170d714cbd8dbc760c1245818e8ed7
+  data.tar.gz: 63995ccbb57fa16e4954cda4ab9610506b0a8ff35084492b520fc18608face35
 SHA512:
-  metadata.gz: 592ecc114de13f0e43bba9193e1ffd3a973c89a43fac3ed1b750b6a70e29b5bf128a05657baf3fc2ccb77134f092efac055651907f01c0ed6d3c00d45a5ebdc9
-  data.tar.gz: a7baed1f1df451e8bd5183fec4631e49c0761e700c4c95fc070389894894a5fb90103d0adce29da797bc1cae72f8a1f21e71da0279e8c18cb62e3b3b5ae05f0a
+  metadata.gz: f3b864db74530840437c0af43475ed0e12a876d6d2cbbd808ecc72722adddbd8d5548c9b777d4e34af5deddf94871567717518f70d787ea7f6a089f2f5b2f4ed
+  data.tar.gz: 4569eed5baa10134e99a0b1404c213e3b35c08ec158e2fa8b006918298750919e8d34074aa10d3b4dfb621bb18cbe68be467d1e7822fb855012cff0f64dcdc7b

data/Changes.md

@@ -2,6 +2,16 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
 
+6.2.2
+---------
+
+- Reduce retry jitter, add jitter to `sidekiq_retry_in` values [#4957]
+- Minimize scheduler load on Redis at scale [#4882]
+- Improve logging of delay jobs [#4904, BuonOno]
+- Minor CSS improvements for buttons and tables, design PRs always welcome!
+- Tweak Web UI `Cache-Control` header [#4966]
+- Rename internal API class `Sidekiq::Job` to `Sidekiq::JobRecord` [#4955]
+
 6.2.1
 ---------
 

data/LICENSE

@@ -6,4 +6,4 @@ for license text.
 
 Sidekiq Pro has a commercial-friendly license allowing private forks
 and modifications of Sidekiq.  Please see https://sidekiq.org/products/pro.html for
-more detail.  You can find the commercial license terms in COMM-LICENSE.
+more detail.  You can find the commercial license terms in COMM-LICENSE.txt.

data/lib/sidekiq/api.rb

@@ -8,7 +8,7 @@ require "base64"
 module Sidekiq
   class Stats
     def initialize
-      fetch_stats!
+      fetch_stats_fast!
     end
 
     def processed
@@ -51,7 +51,8 @@ module Sidekiq
       Sidekiq::Stats::Queues.new.lengths
     end
 
-    def fetch_stats!
+    # O(1) redis calls
+    def fetch_stats_fast!
       pipe1_res = Sidekiq.redis { |conn|
         conn.pipelined do
           conn.get("stat:processed")
@@ -64,25 +65,6 @@ module Sidekiq
         end
       }
 
-      processes = Sidekiq.redis { |conn|
-        conn.sscan_each("processes").to_a
-      }
-
-      queues = Sidekiq.redis { |conn|
-        conn.sscan_each("queues").to_a
-      }
-
-      pipe2_res = Sidekiq.redis { |conn|
-        conn.pipelined do
-          processes.each { |key| conn.hget(key, "busy") }
-          queues.each { |queue| conn.llen("queue:#{queue}") }
-        end
-      }
-
-      s = processes.size
-      workers_size = pipe2_res[0...s].sum(&:to_i)
-      enqueued = pipe2_res[s..-1].sum(&:to_i)
-
       default_queue_latency = if (entry = pipe1_res[6].first)
         job = begin
           Sidekiq.load_json(entry)
@@ -95,6 +77,7 @@ module Sidekiq
       else
         0
       end
+
       @stats = {
         processed: pipe1_res[0].to_i,
         failed: pipe1_res[1].to_i,
@@ -103,10 +86,38 @@ module Sidekiq
         dead_size: pipe1_res[4],
         processes_size: pipe1_res[5],
 
-        default_queue_latency: default_queue_latency,
-        workers_size: workers_size,
-        enqueued: enqueued
+        default_queue_latency: default_queue_latency
+      }
+    end
+
+    # O(number of processes + number of queues) redis calls
+    def fetch_stats_slow!
+      processes = Sidekiq.redis { |conn|
+        conn.sscan_each("processes").to_a
+      }
+
+      queues = Sidekiq.redis { |conn|
+        conn.sscan_each("queues").to_a
       }
+
+      pipe2_res = Sidekiq.redis { |conn|
+        conn.pipelined do
+          processes.each { |key| conn.hget(key, "busy") }
+          queues.each { |queue| conn.llen("queue:#{queue}") }
+        end
+      }
+
+      s = processes.size
+      workers_size = pipe2_res[0...s].sum(&:to_i)
+      enqueued = pipe2_res[s..-1].sum(&:to_i)
+
+      @stats[:workers_size] = workers_size
+      @stats[:enqueued] = enqueued
+    end
+
+    def fetch_stats!
+      fetch_stats_fast!
+      fetch_stats_slow!
     end
 
     def reset(*stats)
@@ -126,7 +137,8 @@ module Sidekiq
     private
 
     def stat(s)
-      @stats[s]
+      fetch_stats_slow! if @stats[s].nil?
+      @stats[s] || raise(ArgumentError, "Unknown stat #{s}")
     end
 
     class Queues
@@ -141,7 +153,7 @@ module Sidekiq
           }
 
           array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
-          Hash[array_of_arrays]
+          array_of_arrays.to_h
         end
       end
     end
@@ -255,7 +267,7 @@ module Sidekiq
         break if entries.empty?
         page += 1
         entries.each do |entry|
-          yield Job.new(entry, @name)
+          yield JobRecord.new(entry, @name)
         end
         deleted_size = initial_size - size
       end
@@ -265,7 +277,7 @@ module Sidekiq
     # Find the job with the given JID within this queue.
     #
     # This is a slow, inefficient operation.  Do not use under
-    # normal conditions.  Sidekiq Pro contains a faster version.
+    # normal conditions.
     def find_job(jid)
       detect { |j| j.jid == jid }
     end
@@ -286,9 +298,9 @@ module Sidekiq
   # sorted set.
   #
   # The job should be considered immutable but may be
-  # removed from the queue via Job#delete.
+  # removed from the queue via JobRecord#delete.
   #
-  class Job
+  class JobRecord
     attr_reader :item
     attr_reader :value
 
@@ -316,21 +328,23 @@ module Sidekiq
 
     def display_class
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
-      @klass ||= case klass
-                 when /\ASidekiq::Extensions::Delayed/
-                   safe_load(args[0], klass) do |target, method, _|
-                     "#{target}.#{method}"
-                   end
-                 when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
-                   job_class = @item["wrapped"] || args[0]
-                   if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
-                     # MailerClass#mailer_method
-                     args[0]["arguments"][0..1].join("#")
-                   else
-                     job_class
-                   end
-                 else
-                   klass
+      @klass ||= self["display_class"] || begin
+        case klass
+        when /\ASidekiq::Extensions::Delayed/
+          safe_load(args[0], klass) do |target, method, _|
+            "#{target}.#{method}"
+          end
+        when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+          job_class = @item["wrapped"] || args[0]
+          if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
+            # MailerClass#mailer_method
+            args[0]["arguments"][0..1].join("#")
+          else
+            job_class
+          end
+        else
+          klass
+        end
       end
     end
 
@@ -443,7 +457,7 @@ module Sidekiq
     end
   end
 
-  class SortedEntry < Job
+  class SortedEntry < JobRecord
     attr_reader :score
     attr_reader :parent
 
@@ -823,12 +837,13 @@ module Sidekiq
     # For Sidekiq Enterprise customers this number (in production) must be
     # less than or equal to your licensed concurrency.
     def total_concurrency
-      sum { |x| x["concurrency"] }
+      sum { |x| x["concurrency"].to_i }
     end
 
-    def total_rss
-      sum { |x| x["rss"] || 0 }
+    def total_rss_in_kb
+      sum { |x| x["rss"].to_i }
     end
+    alias_method :total_rss, :total_rss_in_kb
 
     # Returns the identity of the current cluster leader or "" if no leader.
     # This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
@@ -879,6 +894,10 @@ module Sidekiq
       self["identity"]
     end
 
+    def queues
+      self["queues"]
+    end
+
     def quiet!
       signal("TSTP")
     end
@@ -909,8 +928,8 @@ module Sidekiq
   end
 
   ##
-  # A worker is a thread that is currently processing a job.
-  # Programmatic access to the current active worker set.
+  # The WorkSet stores the work being done by this Sidekiq cluster.
+  # It tracks the process and thread working on each job.
   #
   # WARNING WARNING WARNING
   #
@@ -918,17 +937,17 @@ module Sidekiq
   # If you call #size => 5 and then expect #each to be
   # called 5 times, you're going to have a bad time.
   #
-  #    workers = Sidekiq::Workers.new
-  #    workers.size => 2
-  #    workers.each do |process_id, thread_id, work|
+  #    works = Sidekiq::WorkSet.new
+  #    works.size => 2
+  #    works.each do |process_id, thread_id, work|
   #      # process_id is a unique identifier per Sidekiq process
   #      # thread_id is a unique identifier per thread
   #      # work is a Hash which looks like:
-  #      # { 'queue' => name, 'run_at' => timestamp, 'payload' => msg }
+  #      # { 'queue' => name, 'run_at' => timestamp, 'payload' => job_hash }
   #      # run_at is an epoch Integer.
   #    end
   #
-  class Workers
+  class WorkSet
     include Enumerable
 
     def each(&block)
@@ -975,4 +994,8 @@ module Sidekiq
       end
     end
   end
+  # Since "worker" is a nebulous term, we've deprecated the use of this class name.
+  # Is "worker" a process, a type of job, a thread? Undefined!
+  # WorkSet better describes the data.
+  Workers = WorkSet
 end

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -24,7 +24,9 @@ module Sidekiq
         if marshalled.size > SIZE_LIMIT
           ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
         end
-        @performable.client_push({"class" => @performable, "args" => [marshalled]}.merge(@opts))
+        @performable.client_push({"class" => @performable,
+                                  "args" => [marshalled],
+                                  "display_class" => "#{@target}.#{name}"}.merge(@opts))
       end
     end
   end

data/lib/sidekiq/fetch.rb

@@ -40,7 +40,7 @@ module Sidekiq
       # 4825 Sidekiq Pro with all queues paused will return an
       # empty set of queues with a trailing TIMEOUT value.
       if qs.size <= 1
-        sleep(2)
+        sleep(TIMEOUT)
         return nil
       end
 

data/lib/sidekiq/job.rb

@@ -0,0 +1,8 @@
+require "sidekiq/worker"
+
+module Sidekiq
+  # Sidekiq::Job is a new alias for Sidekiq::Worker, coming in 6.3.0.
+  # You can opt into this by requiring 'sidekiq/job' in your initializer
+  # and then using `include Sidekiq::Job` rather than `Sidekiq::Worker`.
+  Job = Worker
+end

data/lib/sidekiq/job_logger.rb

@@ -38,7 +38,7 @@ module Sidekiq
       # If we're using a wrapper class, like ActiveJob, use the "wrapped"
       # attribute to expose the underlying thing.
       h = {
-        class: job_hash["wrapped"] || job_hash["class"],
+        class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
         jid: job_hash["jid"]
       }
       h[:bid] = job_hash["bid"] if job_hash["bid"]

data/lib/sidekiq/job_retry.rb

@@ -214,16 +214,12 @@ module Sidekiq
     end
 
     def delay_for(worker, count, exception)
+      jitter = rand(10) * (count + 1)
       if worker&.sidekiq_retry_in_block
         custom_retry_in = retry_in(worker, count, exception).to_i
-        return custom_retry_in if custom_retry_in > 0
+        return custom_retry_in + jitter if custom_retry_in > 0
       end
-      seconds_to_delay(count)
-    end
-
-    # delayed_job uses the same basic formula
-    def seconds_to_delay(count)
-      (count**4) + 15 + (rand(30) * (count + 1))
+      (count**4) + 15 + jitter
     end
 
     def retry_in(worker, count, exception)

data/lib/sidekiq/launcher.rb

@@ -238,26 +238,22 @@ module Sidekiq
     end
 
     def to_data
-      @data ||= begin
-        {
-          "hostname" => hostname,
-          "started_at" => Time.now.to_f,
-          "pid" => ::Process.pid,
-          "tag" => @options[:tag] || "",
-          "concurrency" => @options[:concurrency],
-          "queues" => @options[:queues].uniq,
-          "labels" => @options[:labels],
-          "identity" => identity
-        }
-      end
+      @data ||= {
+        "hostname" => hostname,
+        "started_at" => Time.now.to_f,
+        "pid" => ::Process.pid,
+        "tag" => @options[:tag] || "",
+        "concurrency" => @options[:concurrency],
+        "queues" => @options[:queues].uniq,
+        "labels" => @options[:labels],
+        "identity" => identity
+      }
     end
 
     def to_json
-      @json ||= begin
-        # this data changes infrequently so dump it to a string
-        # now so we don't need to dump it every heartbeat.
-        Sidekiq.dump_json(to_data)
-      end
+      # this data changes infrequently so dump it to a string
+      # now so we don't need to dump it every heartbeat.
+      @json ||= Sidekiq.dump_json(to_data)
     end
   end
 end

data/lib/sidekiq/middleware/chain.rb

@@ -90,12 +90,12 @@ module Sidekiq
       end
 
       def add(klass, *args)
-        remove(klass) if exists?(klass)
+        remove(klass)
         entries << Entry.new(klass, *args)
       end
 
       def prepend(klass, *args)
-        remove(klass) if exists?(klass)
+        remove(klass)
         entries.insert(0, Entry.new(klass, *args))
       end
 
@@ -132,7 +132,7 @@ module Sidekiq
       def invoke(*args)
         return yield if empty?
 
-        chain = retrieve.dup
+        chain = retrieve
         traverse_chain = proc do
           if chain.empty?
             yield
@@ -144,6 +144,8 @@ module Sidekiq
       end
     end
 
+    private
+
     class Entry
       attr_reader :klass
 

data/lib/sidekiq/scheduled.rb

@@ -49,6 +49,7 @@ module Sidekiq
         @sleeper = ConnectionPool::TimedStack.new
         @done = false
         @thread = nil
+        @count_calls = 0
       end
 
       # Shut down this instance, will pause until the thread is dead.
@@ -152,8 +153,13 @@ module Sidekiq
       end
 
       def process_count
-        pcount = Sidekiq::ProcessSet.new.size
+        # The work buried within Sidekiq::ProcessSet#cleanup can be
+        # expensive at scale. Cut it down by 90% with this counter.
+        # NB: This method is only called by the scheduler thread so we
+        # don't need to worry about the thread safety of +=.
+        pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
         pcount = 1 if pcount == 0
+        @count_calls += 1
         pcount
       end