sidekiq 6.1.2 → 6.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a80310735fb458e7332a5ecc86072248e0777e1951f77f14965c06313c5b2f8
-  data.tar.gz: bfd09dc57c4506ca9c4d74dd9a38e88ef18700aafaf92814d662c6df4e4b2e5b
+  metadata.gz: 1b0da541124d097e05936cc860a93009ba170d714cbd8dbc760c1245818e8ed7
+  data.tar.gz: 63995ccbb57fa16e4954cda4ab9610506b0a8ff35084492b520fc18608face35
 SHA512:
-  metadata.gz: c0e416fe575874c2051e88c64a6cbb30ad2ba4929ca33487ca1323b1963101d3d08ed43b8edbe874bccfa677521b0745baf589b451c7c56ff41b7bc70ee4f88d
-  data.tar.gz: 4d1cd6d03768f9844bd8edd9bdb545a743084322f4edc01682e04b0990b0fb6aa3f1c6ad4f9a87c1b810a07e03b08eae0405ad2291b42d58eebb257bed7a60f4
+  metadata.gz: f3b864db74530840437c0af43475ed0e12a876d6d2cbbd808ecc72722adddbd8d5548c9b777d4e34af5deddf94871567717518f70d787ea7f6a089f2f5b2f4ed
+  data.tar.gz: 4569eed5baa10134e99a0b1404c213e3b35c08ec158e2fa8b006918298750919e8d34074aa10d3b4dfb621bb18cbe68be467d1e7822fb855012cff0f64dcdc7b

data/Changes.md

@@ -2,6 +2,64 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
 
+6.2.2
+---------
+
+- Reduce retry jitter, add jitter to `sidekiq_retry_in` values [#4957]
+- Minimize scheduler load on Redis at scale [#4882]
+- Improve logging of delay jobs [#4904, BuonOno]
+- Minor CSS improvements for buttons and tables, design PRs always welcome!
+- Tweak Web UI `Cache-Control` header [#4966]
+- Rename internal API class `Sidekiq::Job` to `Sidekiq::JobRecord` [#4955]
+
+6.2.1
+---------
+
+- Update RTT warning logic to handle transient RTT spikes [#4851]
+- Fix very low priority CVE on unescaped queue name [#4852]
+- Add note about sessions and Rails apps in API mode
+
+6.2.0
+---------
+
+- Store Redis RTT and log if poor [#4824]
+- Add process/thread stats to Busy page [#4806]
+- Improve Web UI on mobile devices [#4840]
+- **Refactor Web UI session usage** [#4804]
+  Numerous people have hit "Forbidden" errors and struggled with Sidekiq's
+  Web UI session requirement. If you have code in your initializer for
+  Web sessions, it's quite possible it will need to be removed. Here's
+  an overview:
+```
+Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
+make sure you mount Sidekiq::Web *inside* your routes in `config/routes.rb` so
+Sidekiq can reuse the Rails session:
+
+  Rails.application.routes.draw do
+    mount Sidekiq::Web => "/sidekiq"
+    ....
+  end
+
+If this is a bare Rack app, use a session middleware before Sidekiq::Web:
+
+  # first, use IRB to create a shared secret key for sessions and commit it
+  require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
+
+  # now, update your Rack app to include the secret with a session cookie middleware
+  use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
+  run Sidekiq::Web
+
+If this is a Rails app in API mode, you need to enable sessions.
+
+  https://guides.rubyonrails.org/api_app.html#using-session-middlewares
+```
+
+6.1.3
+---------
+
+- Warn if Redis is configured to evict data under memory pressure [#4752]
+- Add process RSS on the Busy page [#4717]
+
 6.1.2
 ---------
 
@@ -150,7 +208,7 @@ assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs_for(FooMailer).size
 
 This release has major breaking changes.  Read and test carefully in production.
 
-- With Rails 6.0.1+, ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq
+- With Rails 6.0.2+, ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq
   features/internals like the retry subsystem. [#4213, pirj]
 ```ruby
 class MyJob < ActiveJob::Base

data/LICENSE

@@ -6,4 +6,4 @@ for license text.
 
 Sidekiq Pro has a commercial-friendly license allowing private forks
 and modifications of Sidekiq.  Please see https://sidekiq.org/products/pro.html for
-more detail.  You can find the commercial license terms in COMM-LICENSE.
+more detail.  You can find the commercial license terms in COMM-LICENSE.txt.

data/lib/sidekiq/api.rb

@@ -8,7 +8,7 @@ require "base64"
 module Sidekiq
   class Stats
     def initialize
-      fetch_stats!
+      fetch_stats_fast!
     end
 
     def processed
@@ -51,7 +51,8 @@ module Sidekiq
       Sidekiq::Stats::Queues.new.lengths
     end
 
-    def fetch_stats!
+    # O(1) redis calls
+    def fetch_stats_fast!
       pipe1_res = Sidekiq.redis { |conn|
         conn.pipelined do
           conn.get("stat:processed")
@@ -64,6 +65,33 @@ module Sidekiq
         end
       }
 
+      default_queue_latency = if (entry = pipe1_res[6].first)
+        job = begin
+          Sidekiq.load_json(entry)
+        rescue
+          {}
+        end
+        now = Time.now.to_f
+        thence = job["enqueued_at"] || now
+        now - thence
+      else
+        0
+      end
+
+      @stats = {
+        processed: pipe1_res[0].to_i,
+        failed: pipe1_res[1].to_i,
+        scheduled_size: pipe1_res[2],
+        retry_size: pipe1_res[3],
+        dead_size: pipe1_res[4],
+        processes_size: pipe1_res[5],
+
+        default_queue_latency: default_queue_latency
+      }
+    end
+
+    # O(number of processes + number of queues) redis calls
+    def fetch_stats_slow!
       processes = Sidekiq.redis { |conn|
         conn.sscan_each("processes").to_a
       }
@@ -83,30 +111,13 @@ module Sidekiq
       workers_size = pipe2_res[0...s].sum(&:to_i)
       enqueued = pipe2_res[s..-1].sum(&:to_i)
 
-      default_queue_latency = if (entry = pipe1_res[6].first)
-        job = begin
-                Sidekiq.load_json(entry)
-              rescue
-                {}
-              end
-        now = Time.now.to_f
-        thence = job["enqueued_at"] || now
-        now - thence
-      else
-        0
-      end
-      @stats = {
-        processed: pipe1_res[0].to_i,
-        failed: pipe1_res[1].to_i,
-        scheduled_size: pipe1_res[2],
-        retry_size: pipe1_res[3],
-        dead_size: pipe1_res[4],
-        processes_size: pipe1_res[5],
+      @stats[:workers_size] = workers_size
+      @stats[:enqueued] = enqueued
+    end
 
-        default_queue_latency: default_queue_latency,
-        workers_size: workers_size,
-        enqueued: enqueued
-      }
+    def fetch_stats!
+      fetch_stats_fast!
+      fetch_stats_slow!
     end
 
     def reset(*stats)
@@ -126,7 +137,8 @@ module Sidekiq
     private
 
     def stat(s)
-      @stats[s]
+      fetch_stats_slow! if @stats[s].nil?
+      @stats[s] || raise(ArgumentError, "Unknown stat #{s}")
     end
 
     class Queues
@@ -141,7 +153,7 @@ module Sidekiq
           }
 
           array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
-          Hash[array_of_arrays]
+          array_of_arrays.to_h
         end
       end
     end
@@ -255,7 +267,7 @@ module Sidekiq
         break if entries.empty?
         page += 1
         entries.each do |entry|
-          yield Job.new(entry, @name)
+          yield JobRecord.new(entry, @name)
         end
         deleted_size = initial_size - size
       end
@@ -265,7 +277,7 @@ module Sidekiq
     # Find the job with the given JID within this queue.
     #
     # This is a slow, inefficient operation.  Do not use under
-    # normal conditions.  Sidekiq Pro contains a faster version.
+    # normal conditions.
     def find_job(jid)
       detect { |j| j.jid == jid }
     end
@@ -286,9 +298,9 @@ module Sidekiq
   # sorted set.
   #
   # The job should be considered immutable but may be
-  # removed from the queue via Job#delete.
+  # removed from the queue via JobRecord#delete.
   #
-  class Job
+  class JobRecord
     attr_reader :item
     attr_reader :value
 
@@ -316,21 +328,23 @@ module Sidekiq
 
     def display_class
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
-      @klass ||= case klass
-                 when /\ASidekiq::Extensions::Delayed/
-                   safe_load(args[0], klass) do |target, method, _|
-                     "#{target}.#{method}"
-                   end
-                 when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
-                   job_class = @item["wrapped"] || args[0]
-                   if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
-                     # MailerClass#mailer_method
-                     args[0]["arguments"][0..1].join("#")
-                   else
-                     job_class
-                   end
-                 else
-                   klass
+      @klass ||= self["display_class"] || begin
+        case klass
+        when /\ASidekiq::Extensions::Delayed/
+          safe_load(args[0], klass) do |target, method, _|
+            "#{target}.#{method}"
+          end
+        when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+          job_class = @item["wrapped"] || args[0]
+          if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
+            # MailerClass#mailer_method
+            args[0]["arguments"][0..1].join("#")
+          else
+            job_class
+          end
+        else
+          klass
+        end
       end
     end
 
@@ -443,7 +457,7 @@ module Sidekiq
     end
   end
 
-  class SortedEntry < Job
+  class SortedEntry < JobRecord
     attr_reader :score
     attr_reader :parent
 
@@ -791,19 +805,23 @@ module Sidekiq
         # you'll be happier this way
         conn.pipelined do
           procs.each do |key|
-            conn.hmget(key, "info", "busy", "beat", "quiet")
+            conn.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
           end
         end
       }
 
-      result.each do |info, busy, at_s, quiet|
+      result.each do |info, busy, at_s, quiet, rss, rtt|
         # If a process is stopped between when we query Redis for `procs` and
         # when we query for `result`, we will have an item in `result` that is
         # composed of `nil` values.
         next if info.nil?
 
         hash = Sidekiq.load_json(info)
-        yield Process.new(hash.merge("busy" => busy.to_i, "beat" => at_s.to_f, "quiet" => quiet))
+        yield Process.new(hash.merge("busy" => busy.to_i,
+                                     "beat" => at_s.to_f,
+                                     "quiet" => quiet,
+                                     "rss" => rss.to_i,
+                                     "rtt_us" => rtt.to_i))
       end
     end
 
@@ -815,6 +833,18 @@ module Sidekiq
       Sidekiq.redis { |conn| conn.scard("processes") }
     end
 
+    # Total number of threads available to execute jobs.
+    # For Sidekiq Enterprise customers this number (in production) must be
+    # less than or equal to your licensed concurrency.
+    def total_concurrency
+      sum { |x| x["concurrency"].to_i }
+    end
+
+    def total_rss_in_kb
+      sum { |x| x["rss"].to_i }
+    end
+    alias_method :total_rss, :total_rss_in_kb
+
     # Returns the identity of the current cluster leader or "" if no leader.
     # This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
     # or Sidekiq Pro.
@@ -864,6 +894,10 @@ module Sidekiq
       self["identity"]
     end
 
+    def queues
+      self["queues"]
+    end
+
     def quiet!
       signal("TSTP")
     end
@@ -894,8 +928,8 @@ module Sidekiq
   end
 
   ##
-  # A worker is a thread that is currently processing a job.
-  # Programmatic access to the current active worker set.
+  # The WorkSet stores the work being done by this Sidekiq cluster.
+  # It tracks the process and thread working on each job.
   #
   # WARNING WARNING WARNING
   #
@@ -903,17 +937,17 @@ module Sidekiq
   # If you call #size => 5 and then expect #each to be
   # called 5 times, you're going to have a bad time.
   #
-  #    workers = Sidekiq::Workers.new
-  #    workers.size => 2
-  #    workers.each do |process_id, thread_id, work|
+  #    works = Sidekiq::WorkSet.new
+  #    works.size => 2
+  #    works.each do |process_id, thread_id, work|
   #      # process_id is a unique identifier per Sidekiq process
   #      # thread_id is a unique identifier per thread
   #      # work is a Hash which looks like:
-  #      # { 'queue' => name, 'run_at' => timestamp, 'payload' => msg }
+  #      # { 'queue' => name, 'run_at' => timestamp, 'payload' => job_hash }
   #      # run_at is an epoch Integer.
   #    end
   #
-  class Workers
+  class WorkSet
     include Enumerable
 
     def each(&block)
@@ -960,4 +994,8 @@ module Sidekiq
       end
     end
   end
+  # Since "worker" is a nebulous term, we've deprecated the use of this class name.
+  # Is "worker" a process, a type of job, a thread? Undefined!
+  # WorkSet better describes the data.
+  Workers = WorkSet
 end

data/lib/sidekiq/cli.rb

@@ -59,9 +59,22 @@ module Sidekiq
 
       # touch the connection pool so it is created before we
       # fire startup and start multithreading.
-      ver = Sidekiq.redis_info["redis_version"]
+      info = Sidekiq.redis_info
+      ver = info["redis_version"]
       raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
 
+      maxmemory_policy = info["maxmemory_policy"]
+      if maxmemory_policy != "noeviction"
+        logger.warn <<~EOM
+
+
+          WARNING: Your Redis instance will evict Sidekiq data under heavy load.
+          The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
+          See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
+
+        EOM
+      end
+
       # Since the user can pass us a connection pool explicitly in the initializer, we
       # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
       cursize = Sidekiq.redis_pool.size

data/lib/sidekiq/client.rb

@@ -19,7 +19,7 @@ module Sidekiq
     #
     def middleware(&block)
       @chain ||= Sidekiq.client_middleware
-      if block_given?
+      if block
         @chain = @chain.dup
         yield @chain
       end
@@ -228,10 +228,6 @@ module Sidekiq
     end
 
     def normalize_item(item)
-      # 6.0.0 push_bulk bug, #4321
-      # TODO Remove after a while...
-      item.delete("at") if item.key?("at") && item["at"].nil?
-
       validate(item)
       # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
 

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -24,7 +24,9 @@ module Sidekiq
         if marshalled.size > SIZE_LIMIT
           ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
         end
-        @performable.client_push({"class" => @performable, "args" => [marshalled]}.merge(@opts))
+        @performable.client_push({"class" => @performable,
+                                  "args" => [marshalled],
+                                  "display_class" => "#{@target}.#{name}"}.merge(@opts))
       end
     end
   end

data/lib/sidekiq/fetch.rb

@@ -36,7 +36,15 @@ module Sidekiq
     end
 
     def retrieve_work
-      work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
+      qs = queues_cmd
+      # 4825 Sidekiq Pro with all queues paused will return an
+      # empty set of queues with a trailing TIMEOUT value.
+      if qs.size <= 1
+        sleep(TIMEOUT)
+        return nil
+      end
+
+      work = Sidekiq.redis { |conn| conn.brpop(*qs) }
       UnitOfWork.new(*work) if work
     end
 

data/lib/sidekiq/job.rb

@@ -0,0 +1,8 @@
+require "sidekiq/worker"
+
+module Sidekiq
+  # Sidekiq::Job is a new alias for Sidekiq::Worker, coming in 6.3.0.
+  # You can opt into this by requiring 'sidekiq/job' in your initializer
+  # and then using `include Sidekiq::Job` rather than `Sidekiq::Worker`.
+  Job = Worker
+end