sidekiq 7.1.6 → 7.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b50ab03f32263dd24ee37ce2ddadc7a4e5e226860146c1a677262dd3a417329d
-  data.tar.gz: 0af1051781796465d5ce5dd0d0e454016fdc04b7cb5fcf0b8d62e447faff11ed
+  metadata.gz: 6c43e6b585c25dcfc8ef8364bb36cf74f9167b981ad03faa3a8d76e0d45ebe55
+  data.tar.gz: d8c65dc03008f7280b36af94db753d4c7f68267c2eb0d78cd018322887aabbb0
 SHA512:
-  metadata.gz: f2e84e49ceb024e8cb24a7213ffe81e40ed9423d646e39778ed51049bf89a04c28890a21e93eeb38bb9daefe88b94b150d71a0cdf80ecdad712a2167dd0a3421
-  data.tar.gz: 39ffc3e4eee3e2c27a3f774f2a2120c50a52ab9e2049b0fc925123c622553bf4fd19512c16e63882031909bba6ea5b2d15cf856256a3c453d080e89253196fc6
+  metadata.gz: d2687692b873ab82bda2ad32e9be795150cd0a8d3d330bc19f5b509ba729bef33189e06ebac86b1906c2682187391d6cf0d532e47d03fcbea83058109c5816ef
+  data.tar.gz: 431a482baeb03fc4de50fbdfba8717fc332a9d6564fde98a77699a7bd174fa3194431385951cf689c64e04853039c95fcf287084f283e8d381b3b37d5bc665e0

data/Changes.md

@@ -2,6 +2,69 @@
 
 [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
 
+7.2.4
+----------
+
+- Fix XSS in metrics filtering introduced in 7.2.0, CVE-2024-32887
+  Thanks to @UmerAdeemCheema for the security report.
+
+7.2.3
+----------
+
+- [Support Dragonfly.io](https://www.mikeperham.com/2024/02/01/supporting-dragonfly/) as an alternative Redis implementation
+- Fix error unpacking some compressed error backtraces [#6241]
+- Fix potential heartbeat data leak [#6227]
+- Add ability to find a currently running work by jid [#6212, fatkodima]
+
+7.2.2
+----------
+
+- Add `Process.warmup` call in Ruby 3.3+
+- Batch jobs now skip transactional push [#6160]
+
+7.2.1
+----------
+
+- Add `Sidekiq::Work` type which replaces the raw Hash as the third parameter in
+  `Sidekiq::WorkSet#each { |pid, tid, hash| ... }` [#6145]
+- **DEPRECATED**: direct access to the attributes within the `hash` block parameter above.
+  The `Sidekiq::Work` instance contains accessor methods to get at the same data, e.g.
+```ruby
+work["queue"] # Old
+work.queue # New
+```
+- Fix Ruby 3.3 warnings around `base64` gem [#6151, earlopain]
+
+7.2.0
+----------
+
+- `sidekiq_retries_exhausted` can return `:discard` to avoid the deadset
+  and all death handlers [#6091]
+- Metrics filtering by job class in Web UI [#5974]
+- Better readability and formatting for numbers within the Web UI [#6080]
+- Add explicit error if user code tries to nest test modes [#6078]
+```ruby
+Sidekiq::Testing.inline! # global setting
+Sidekiq::Testing.fake! do # override within block
+  # ok
+  Sidekiq::Testing.inline! do # can't override the override
+    # not ok, nested
+  end
+end
+```
+- **SECURITY** Forbid inline JavaScript execution in Web UI [#6074]
+- Adjust redis-client adapter to avoid `method_missing` [#6083]
+  This can result in app code breaking if your app's Redis API usage was
+  depending on Sidekiq's adapter to correct invalid redis-client API usage.
+  One example:
+```ruby
+# bad, not redis-client native
+# Unsupported command argument type: TrueClass (TypeError)
+Sidekiq.redis { |c| c.set("key", "value", nx: true, ex: 15) }
+# good
+Sidekiq.redis { |c| c.set("key", "value", "nx", "ex", 15) }
+```
+
 7.1.6
 ----------
 
@@ -150,6 +213,11 @@ end
 - Job Execution metrics!!!
 - See `docs/7.0-Upgrade.md` for release notes
 
+6.5.{10,11,12}
+----------
+
+- Fixes for Rails 7.1 [#6067, #6070]
+
 6.5.9
 ----------
 

data/README.md

@@ -14,11 +14,11 @@ Rails to make background processing dead simple.
 Requirements
 -----------------
 
-- Redis: 6.2+
+- Redis: Redis 6.2+ or Dragonfly 1.13+
 - Ruby: MRI 2.7+ or JRuby 9.3+.
 
 Sidekiq 7.0 supports Rails 6.0+ but does not require it.
-
+As of 7.2, Sidekiq supports Dragonfly as an alternative to Redis for data storage.
 
 Installation
 -----------------

data/bin/multi_queue_bench

@@ -0,0 +1,271 @@
+#!/usr/bin/env ruby
+
+#
+# bin/bench is a helpful script to load test and
+# performance tune Sidekiq's core. It's a configurable script,
+# which accepts the following parameters as ENV variables.
+#
+# QUEUES
+# Number of queues to consume from. Default is 8
+#
+# PROCESSES
+# The number of processes this benchmark will create. Each process, consumes
+# from one of the available queues. When processes are more than the number of
+# queues, they are distributed to processes in round robin. Default is 8
+#
+# ELEMENTS
+# Number of jobs to push to each queue. Default is 1000
+#
+# ITERATIONS
+# Each queue pushes ITERATIONS times ELEMENTS jobs. Default is 1000
+#
+# PORT
+# The port of the Dragonfly instance. Default is 6379
+#
+# IP
+# The ip of the Dragonfly instance. Default is 127.0.0.1
+#
+# Example Usage:
+#
+# > RUBY_YJIT_ENABLE=1 THREADS=10 PROCESSES=8 QUEUES=8 bin/multi_queue_bench
+#
+# None of this script is considered a public API and may change over time.
+#
+
+# Quiet some warnings we see when running in warning mode:
+# RUBYOPT=-w bundle exec sidekiq
+$TESTING = false
+puts RUBY_DESCRIPTION
+
+require "bundler/setup"
+Bundler.require(:default, :load_test)
+
+class LoadWorker
+  include Sidekiq::Job
+  sidekiq_options retry: 1
+  sidekiq_retry_in do |x|
+    1
+  end
+
+  def perform(idx, ts = nil)
+    puts(Time.now.to_f - ts) if !ts.nil?
+    # raise idx.to_s if idx % 100 == 1
+  end
+end
+
+def Process.rss
+  `ps -o rss= -p #{Process.pid}`.chomp.to_i
+end
+
+$iterations = ENV["ITERATIONS"] ? Integer(ENV["ITERATIONS"]) : 1_000
+$elements = ENV["ELEMENTS"] ? Integer(ENV["ELEMENTS"]) : 1_000
+$port = ENV["PORT"] ? Integer(ENV["PORT"]) : 6379
+$ip = ENV["IP"] ? String(ENV["IP"]) : "127.0.0.1"
+
+class Loader
+  def initialize
+    @iter =  $iterations
+    @count = $elements
+  end
+
+  def configure(queue)
+    @x = Sidekiq.configure_embed do |config|
+      config.redis = {db: 0, host: $ip, port: $port}
+      config.concurrency = Integer(ENV.fetch("THREADS", "30"))
+      config.queues = queue
+      config.logger.level = Logger::WARN
+      config.average_scheduled_poll_interval = 2
+      config.reliable! if defined?(Sidekiq::Pro)
+    end
+
+    @self_read, @self_write = IO.pipe
+    %w[INT TERM TSTP TTIN].each do |sig|
+      trap sig do
+        @self_write.puts(sig)
+      end
+    rescue ArgumentError
+      puts "Signal #{sig} not supported"
+    end
+  end
+
+  def handle_signal(sig)
+    launcher = @x
+    Sidekiq.logger.debug "Got #{sig} signal"
+    case sig
+    when "INT"
+      # Handle Ctrl-C in JRuby like MRI
+      # http://jira.codehaus.org/browse/JRUBY-4637
+      raise Interrupt
+    when "TERM"
+      # Heroku sends TERM and then waits 30 seconds for process to exit.
+      raise Interrupt
+    when "TSTP"
+      Sidekiq.logger.info "Received TSTP, no longer accepting new work"
+      launcher.quiet
+    when "TTIN"
+      Thread.list.each do |thread|
+        Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["label"]}"
+        if thread.backtrace
+          Sidekiq.logger.warn thread.backtrace.join("\n")
+        else
+          Sidekiq.logger.warn "<no backtrace available>"
+        end
+      end
+    end
+  end
+
+  def setup(queue)
+    Sidekiq.logger.error("Setup RSS: #{Process.rss}")
+    Sidekiq.logger.error("Pushing work to queue: #{queue}")
+    start = Time.now
+    @iter.times do
+      arr = Array.new(@count) { |idx| [idx] }
+      # Sidekiq always prepends "queue:" to the queue name,
+      # that's why we pass 'q1', 'q2', etc instead of 'queue:q1'
+      Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr, "queue" => queue)
+      $stdout.write "."
+    end
+    puts "Done"
+  end
+
+  def monitor_single(queue)
+    q = "queue:#{queue}"
+    @monitor_single = Thread.new do
+      GC.start
+      loop do
+        sleep 0.2
+        total = Sidekiq.redis do |conn|
+          conn.llen q
+        end
+
+        if total == 0
+          sleep 0.1
+          @x.stop
+          Process.kill("INT", $$)
+          break
+        end
+
+      end
+    end
+  end
+
+  def monitor_all(queues)
+    @monitor_all = Thread.new do
+      GC.start
+      loop do
+        sleep 0.2
+        qsize = 0
+        queues.each do |q|
+          tmp = Sidekiq.redis do |conn|
+            conn.llen q
+          end
+          qsize = qsize + tmp
+        end
+        total = qsize
+
+        if total == 0
+          ending = Time.now - @start
+          size = @iter * @count * queues.length()
+          Sidekiq.logger.error("Done, #{size} jobs in #{ending} sec, #{(size / ending).to_i} jobs/sec")
+          Sidekiq.logger.error("Ending RSS: #{Process.rss}")
+
+          sleep 0.1
+          @x.stop
+          Process.kill("INT", $$)
+          break
+        end
+      end
+    end
+  end
+
+  def run(queues, queue, monitor_all_queues)
+    Sidekiq.logger.warn("Consuming from #{queue}")
+    if monitor_all_queues
+      monitor_all(queues)
+    else
+      monitor_single(queue)
+    end
+
+    @start = Time.now
+    @x.run
+
+    while (readable_io = IO.select([@self_read]))
+      signal = readable_io.first[0].gets.strip
+      handle_signal(signal)
+    end
+    # normal
+  rescue Interrupt
+  rescue => e
+    raise e if $DEBUG
+    warn e.message
+    warn e.backtrace.join("\n")
+    exit 1
+  ensure
+    @x.stop
+  end
+end
+
+def setup(queue)
+  ll = Loader.new
+  ll.configure(queue)
+  ll.setup(queue)
+end
+
+def consume(queues, queue, monitor_all_queues)
+  ll = Loader.new
+  ll.configure(queue)
+  ll.run(queues, queue, monitor_all_queues)
+end
+
+# We assign one queue to each sidekiq process
+def run(number_of_processes, total_queues)
+  read_stream, write_stream = IO.pipe
+
+  queues = []
+  (0..total_queues-1).each do |idx|
+    queues.push("queue:q#{idx}")
+  end
+
+  Sidekiq.logger.info("Queues are: #{queues}")
+
+  # Produce
+  start = Time.now
+  (0..total_queues-1).each do |idx|
+    Process.fork do
+      queue_num = "q#{idx}"
+      setup(queue_num)
+    end
+  end
+
+  queue_sz = $iterations * $elements * total_queues
+  Process.waitall
+
+  ending = Time.now - start
+  #Sidekiq.logger.info("Pushed #{queue_sz} in #{ending} secs")
+
+  # Consume
+  (0..number_of_processes-1).each do |idx|
+    Process.fork do
+      # First process only consumes from it's own queue but monitors all queues.
+      # It works as a synchronization point. Once all processes finish
+      # (that is, when all queues are emptied) it prints the the stats.
+      if idx == 0
+        queue = "q#{idx}"
+        consume(queues, queue, true)
+      else
+        queue = "q#{idx % total_queues}"
+        consume(queues, queue, false)
+      end
+    end
+  end
+
+  Process.waitall
+  write_stream.close
+  results = read_stream.read
+  read_stream.close
+end
+
+$total_processes = ENV["PROCESSES"] ? Integer(ENV["PROCESSES"]) : 8;
+$total_queues = ENV["QUEUES"] ? Integer(ENV["QUEUES"]) : 8;
+
+run($total_processes, $total_queues)

data/lib/sidekiq/api.rb

@@ -4,7 +4,6 @@ require "sidekiq"
 
 require "zlib"
 require "set"
-require "base64"
 
 require "sidekiq/metrics/query"
 
@@ -491,8 +490,8 @@ module Sidekiq
     end
 
     def uncompress_backtrace(backtrace)
-      decoded = Base64.decode64(backtrace)
-      uncompressed = Zlib::Inflate.inflate(decoded)
+      strict_base64_decoded = backtrace.unpack1("m")
+      uncompressed = Zlib::Inflate.inflate(strict_base64_decoded)
       Sidekiq.load_json(uncompressed)
     end
   end
@@ -679,7 +678,7 @@ module Sidekiq
         range_start = page * page_size + offset_size
         range_end = range_start + page_size - 1
         elements = Sidekiq.redis { |conn|
-          conn.zrange name, range_start, range_end, withscores: true
+          conn.zrange name, range_start, range_end, "withscores"
         }
         break if elements.empty?
         page -= 1
@@ -706,7 +705,7 @@ module Sidekiq
         end
 
       elements = Sidekiq.redis { |conn|
-        conn.zrange(name, begin_score, end_score, "BYSCORE", withscores: true)
+        conn.zrange(name, begin_score, end_score, "BYSCORE", "withscores")
       }
 
       elements.each_with_object([]) do |element, result|
@@ -774,7 +773,7 @@ module Sidekiq
   #
   class ScheduledSet < JobSet
     def initialize
-      super "schedule"
+      super("schedule")
     end
   end
 
@@ -788,7 +787,7 @@ module Sidekiq
   #
   class RetrySet < JobSet
     def initialize
-      super "retry"
+      super("retry")
     end
 
     # Enqueues all jobs pending within the retry set.
@@ -809,7 +808,7 @@ module Sidekiq
   #
   class DeadSet < JobSet
     def initialize
-      super "dead"
+      super("dead")
     end
 
     # Add the given job to the Dead set.
@@ -881,7 +880,7 @@ module Sidekiq
     # @api private
     def cleanup
       # dont run cleanup more than once per minute
-      return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
+      return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", "NX", "EX", "60") }
 
       count = 0
       Sidekiq.redis do |conn|
@@ -1110,11 +1109,11 @@ module Sidekiq
 
       procs.zip(all_works).each do |key, workers|
         workers.each_pair do |tid, json|
-          results << [key, tid, Sidekiq.load_json(json)] unless json.empty?
+          results << [key, tid, Sidekiq::Work.new(key, tid, Sidekiq.load_json(json))] unless json.empty?
         end
       end
 
-      results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
+      results.sort_by { |(_, _, hsh)| hsh.raw("run_at") }.each(&block)
     end
 
     # Note that #size is only as accurate as Sidekiq's heartbeat,
@@ -1137,7 +1136,74 @@ module Sidekiq
         end
       end
     end
+
+    ##
+    # Find the work which represents a job with the given JID.
+    # *This is a slow O(n) operation*.  Do not use for app logic.
+    #
+    # @param jid [String] the job identifier
+    # @return [Sidekiq::Work] the work or nil
+    def find_work_by_jid(jid)
+      each do |_process_id, _thread_id, work|
+        job = work.job
+        return work if job.jid == jid
+      end
+      nil
+    end
   end
+
+  # Sidekiq::Work represents a job which is currently executing.
+  class Work
+    attr_reader :process_id
+    attr_reader :thread_id
+
+    def initialize(pid, tid, hsh)
+      @process_id = pid
+      @thread_id = tid
+      @hsh = hsh
+      @job = nil
+    end
+
+    def queue
+      @hsh["queue"]
+    end
+
+    def run_at
+      Time.at(@hsh["run_at"])
+    end
+
+    def job
+      @job ||= Sidekiq::JobRecord.new(@hsh["payload"])
+    end
+
+    def payload
+      @hsh["payload"]
+    end
+
+    # deprecated
+    def [](key)
+      kwargs = {uplevel: 1}
+      kwargs[:category] = :deprecated if RUBY_VERSION > "3.0" # TODO
+      warn("Direct access to `Sidekiq::Work` attributes is deprecated, please use `#payload`, `#queue`, `#run_at` or `#job` instead", **kwargs)
+
+      @hsh[key]
+    end
+
+    # :nodoc:
+    # @api private
+    def raw(name)
+      @hsh[name]
+    end
+
+    def method_missing(*all)
+      @hsh.send(*all)
+    end
+
+    def respond_to_missing?(name)
+      @hsh.respond_to?(name)
+    end
+  end
+
   # Since "worker" is a nebulous term, we've deprecated the use of this class name.
   # Is "worker" a process, a type of job, a thread? Undefined!
   # WorkSet better describes the data.

data/lib/sidekiq/cli.rb

@@ -38,7 +38,7 @@ module Sidekiq # :nodoc:
     # Code within this method is not tested because it alters
     # global process state irreversibly.  PRs which improve the
     # test coverage of Sidekiq::CLI are welcomed.
-    def run(boot_app: true)
+    def run(boot_app: true, warmup: true)
       boot_application if boot_app
 
       if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
@@ -101,6 +101,8 @@ module Sidekiq # :nodoc:
       # Touch middleware so it isn't lazy loaded by multiple threads, #3043
       @config.server_middleware
 
+      ::Process.warmup if warmup && ::Process.respond_to?(:warmup)
+
       # Before this point, the process is initializing with just the main thread.
       # Starting here the process will now have multiple threads running.
       fire_event(:startup, reverse: false, reraise: true)

data/lib/sidekiq/config.rb

@@ -258,9 +258,9 @@ module Sidekiq
       @logger = logger
     end
 
-    private def arity(handler)
-      return handler.arity if handler.is_a?(Proc)
-      handler.method(:call).arity
+    private def parameter_size(handler)
+      target = handler.is_a?(Proc) ? handler : handler.method(:call)
+      target.parameters.size
     end
 
     # INTERNAL USE ONLY
@@ -269,7 +269,7 @@ module Sidekiq
         p ["!!!!!", ex]
       end
       @options[:error_handlers].each do |handler|
-        if arity(handler) == 2
+        if parameter_size(handler) == 2
           # TODO Remove in 8.0
           logger.info { "DEPRECATION: Sidekiq exception handlers now take three arguments, see #{handler}" }
           handler.call(ex, {_config: self}.merge(ctx))

data/lib/sidekiq/deploy.rb

@@ -34,7 +34,7 @@ module Sidekiq
       # handle an very common error in marking deploys:
       # having every process mark its deploy, leading
       # to N marks for each deploy. Instead we round the time
-      # to the minute so that multple marks within that minute
+      # to the minute so that multiple marks within that minute
       # will all naturally rollup into one mark per minute.
       whence = at.utc
       floor = Time.utc(whence.year, whence.month, whence.mday, whence.hour, whence.min, 0)
@@ -44,7 +44,7 @@ module Sidekiq
 
       @pool.with do |c|
         # only allow one deploy mark for a given label for the next minute
-        lock = c.set("deploylock-#{label}", stamp, nx: true, ex: 60)
+        lock = c.set("deploylock-#{label}", stamp, "nx", "ex", "60")
         if lock
           c.multi do |pipe|
             pipe.hsetnx(key, stamp, label)

data/lib/sidekiq/job.rb

@@ -109,7 +109,7 @@ module Sidekiq
               m = "#{name}="
               undef_method(m) if method_defined?(m) || private_method_defined?(m)
             end
-            define_singleton_method("#{name}=") do |val|
+            define_singleton_method(:"#{name}=") do |val|
               singleton_class.class_eval do
                 ACCESSOR_MUTEX.synchronize do
                   undef_method(synchronized_getter) if method_defined?(synchronized_getter) || private_method_defined?(synchronized_getter)

data/lib/sidekiq/job_retry.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "zlib"
-require "base64"
 require "sidekiq/component"
 
 module Sidekiq
@@ -226,7 +225,7 @@ module Sidekiq
     end
 
     def retries_exhausted(jobinst, msg, exception)
-      begin
+      rv = begin
         block = jobinst&.sidekiq_retries_exhausted_block
 
         # the sidekiq_retries_exhausted_block can be defined in a wrapped class (ActiveJob for instance)
@@ -239,6 +238,7 @@ module Sidekiq
         handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
       end
 
+      return if rv == :discard # poof!
       send_to_morgue(msg) unless msg["dead"] == false
 
       @capsule.config.death_handlers.each do |handler|
@@ -294,7 +294,7 @@ module Sidekiq
     def compress_backtrace(backtrace)
       serialized = Sidekiq.dump_json(backtrace)
       compressed = Zlib::Deflate.deflate(serialized)
-      Base64.encode64(compressed)
+      [compressed].pack("m0") # Base64.strict_encode64
     end
   end
 end

data/lib/sidekiq/launcher.rb

@@ -145,15 +145,17 @@ module Sidekiq
         flush_stats
 
         curstate = Processor::WORK_STATE.dup
+        curstate.transform_values! { |val| Sidekiq.dump_json(val) }
+
         redis do |conn|
           # work is the current set of executing jobs
           work_key = "#{key}:work"
-          conn.pipelined do |transaction|
+          conn.multi do |transaction|
             transaction.unlink(work_key)
-            curstate.each_pair do |tid, hash|
-              transaction.hset(work_key, tid, Sidekiq.dump_json(hash))
+            if curstate.size > 0
+              transaction.hset(work_key, curstate)
+              transaction.expire(work_key, 60)
             end
-            transaction.expire(work_key, 60)
           end
         end
 

data/lib/sidekiq/logger.rb

@@ -36,7 +36,7 @@ module Sidekiq
     end
 
     LEVELS.each do |level, numeric_level|
-      define_method("#{level}?") do
+      define_method(:"#{level}?") do
         local_level.nil? ? super() : local_level <= numeric_level
       end
     end