sidekiq 7.2.0 → 7.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30f824346db9b0ebf8ee13c6ac0101494e5fd6d05b4ed2ef3ad97c5b17ccbc10
-  data.tar.gz: bed22f02925116256550bbc34ef9decd70bdbca356d5d61abedd4d14a7dcac45
+  metadata.gz: 34d69bb09eb124a8fd4710af9972bdb6a0cd74429bc3e61cdccdae683a4a0203
+  data.tar.gz: 8a37c7f172e4efabb061f980a73547d7245585574133b07d159f0412e49dd362
 SHA512:
-  metadata.gz: 347e82cf6f215a1e4bd09c3f12d382be79d96bb83ccd1d09f928db19b936c2dd72c0f2e3a8988160086da7928a7911d84eddd005428c7b6a337763c4b60a3492
-  data.tar.gz: ef0a03f45d4d35e832f36b7a09ae0694838cd0cf3582dc20d53956bb5a61ba07811d410600b43a9bdf777e68dd2549910a125d23885afca8388b8513bb7ac8d1
+  metadata.gz: dbb63070d419ad3d2b8192b2864ff4608288c00fe51c7e79e4a3faafd33665cce0c92ece38f9cd2a30ae4b7ff7e66debaf894198b054b88c5d0b5d5ee3790153
+  data.tar.gz: 8ba7a30ebdd19734a7cc64031a6eb63e4f317de0ae652bed324c272fd57f422186add097b38ab88aaa2a0a925de6b0845d18739691cbd9e9f2e90f3d79bb9e0a

data/Changes.md

@@ -2,6 +2,19 @@
 
 [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
 
+7.2.1
+----------
+
+- Add `Sidekiq::Work` type which replaces the raw Hash as the third parameter in
+  `Sidekiq::WorkSet#each { |pid, tid, hash| ... }` [#6145]
+- **DEPRECATED**: direct access to the attributes within the `hash` block parameter above.
+  The `Sidekiq::Work` instance contains accessor methods to get at the same data, e.g.
+```ruby
+work["queue"] # Old
+work.queue # New
+```
+- Fix Ruby 3.3 warnings around `base64` gem [#6151, earlopain]
+
 7.2.0
 ----------
 

data/bin/multi_queue_bench

@@ -0,0 +1,268 @@
+#!/usr/bin/env ruby
+
+#
+# bin/bench is a helpful script to load test and
+# performance tune Sidekiq's core. It's a configurable script,
+# which accepts the following parameters as ENV variables.
+#
+# QUEUES
+# Number of queues to consume from. Default is 8
+#
+# PROCESSES
+# The number of processes this benchmark will create. Each process, consumes
+# from one of the available queues. When processes are more than the number of
+# queues, they are distributed to processes in round robin. Default is 8
+#
+# ELEMENTS
+# Number of jobs to push to each queue. Default is 1000
+#
+# ITERATIONS
+# Each queue pushes ITERATIONS times ELEMENTS jobs. Default is 1000
+#
+# PORT
+# The port of the Dragonfly instance. Default is 6379
+#
+# IP
+# The ip of the Dragonfly instance. Default is 127.0.0.1
+#
+# Example Usage:
+#
+# > RUBY_YJIT_ENABLE=1 THREADS=10 PROCESSES=8 QUEUES=8 bin/multi_queue_bench
+#
+# None of this script is considered a public API and may change over time.
+#
+
+# Quiet some warnings we see when running in warning mode:
+# RUBYOPT=-w bundle exec sidekiq
+$TESTING = false
+puts RUBY_DESCRIPTION
+
+require "bundler/setup"
+Bundler.require(:default, :load_test)
+
+class LoadWorker
+  include Sidekiq::Job
+  sidekiq_options retry: 1
+  sidekiq_retry_in do |x|
+    1
+  end
+
+  def perform(idx, ts = nil)
+    puts(Time.now.to_f - ts) if !ts.nil?
+    # raise idx.to_s if idx % 100 == 1
+  end
+end
+
+def Process.rss
+  `ps -o rss= -p #{Process.pid}`.chomp.to_i
+end
+
+$iterations = ENV["ITERATIONS"] ? Integer(ENV["ITERATIONS"]) : 1_000
+$elements = ENV["ELEMENTS"] ? Integer(ENV["ELEMENTS"]) : 1_000
+$port = ENV["PORT"] ? Integer(ENV["PORT"]) : 6379
+$ip = ENV["IP"] ? String(ENV["IP"]) : "127.0.0.1"
+
+class Loader
+  def initialize
+    @iter =  $iterations
+    @count = $elements
+  end
+
+  def configure(queue)
+    @x = Sidekiq.configure_embed do |config|
+      config.redis = {db: 0, host: $ip, port: $port}
+      config.concurrency = Integer(ENV.fetch("THREADS", "30"))
+      config.queues = queue
+      config.logger.level = Logger::WARN
+      config.average_scheduled_poll_interval = 2
+      config.reliable! if defined?(Sidekiq::Pro)
+    end
+
+    @self_read, @self_write = IO.pipe
+    %w[INT TERM TSTP TTIN].each do |sig|
+      trap sig do
+        @self_write.puts(sig)
+      end
+    rescue ArgumentError
+      puts "Signal #{sig} not supported"
+    end
+  end
+
+  def handle_signal(sig)
+    launcher = @x
+    Sidekiq.logger.debug "Got #{sig} signal"
+    case sig
+    when "INT"
+      # Handle Ctrl-C in JRuby like MRI
+      # http://jira.codehaus.org/browse/JRUBY-4637
+      raise Interrupt
+    when "TERM"
+      # Heroku sends TERM and then waits 30 seconds for process to exit.
+      raise Interrupt
+    when "TSTP"
+      Sidekiq.logger.info "Received TSTP, no longer accepting new work"
+      launcher.quiet
+    when "TTIN"
+      Thread.list.each do |thread|
+        Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["label"]}"
+        if thread.backtrace
+          Sidekiq.logger.warn thread.backtrace.join("\n")
+        else
+          Sidekiq.logger.warn "<no backtrace available>"
+        end
+      end
+    end
+  end
+
+  def setup(queue)
+    Sidekiq.logger.error("Setup RSS: #{Process.rss}")
+    Sidekiq.logger.error("Pushing work to queue: #{queue}")
+    start = Time.now
+    @iter.times do
+      arr = Array.new(@count) { |idx| [idx] }
+      #always prepends by queue:: that's why we pass 'q1, q2 etc' instead of `queue::q1`
+      Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr, "queue" => queue)
+    end
+  end
+
+  def monitor_single(queue)
+    q = "queue:#{queue}"
+    @monitor_single = Thread.new do
+      GC.start
+      loop do
+        sleep 0.2
+        total = Sidekiq.redis do |conn|
+          conn.llen q
+        end
+
+        if total == 0
+          sleep 0.1
+          @x.stop
+          Process.kill("INT", $$)
+          break
+        end
+
+      end
+    end
+  end
+
+  def monitor_all(queues)
+    @monitor_all = Thread.new do
+      GC.start
+      loop do
+        sleep 0.2
+        qsize = 0
+        queues.each do |q|
+          tmp = Sidekiq.redis do |conn|
+            conn.llen q
+          end
+          qsize = qsize + tmp
+        end
+        total = qsize
+
+        if total == 0
+          ending = Time.now - @start
+          size = @iter * @count * queues.length()
+          Sidekiq.logger.error("Done, #{size} jobs in #{ending} sec, #{(size / ending).to_i} jobs/sec")
+          Sidekiq.logger.error("Ending RSS: #{Process.rss}")
+
+          sleep 0.1
+          @x.stop
+          Process.kill("INT", $$)
+          break
+        end
+      end
+    end
+  end
+
+  def run(queues, queue, monitor_all_queues)
+    #Sidekiq.logger.warn("Consuming from #{queue}")
+    if monitor_all_queues
+      monitor_all(queues)
+    else
+      monitor_single(queue)
+    end
+
+    @start = Time.now
+    @x.run
+
+    while (readable_io = IO.select([@self_read]))
+      signal = readable_io.first[0].gets.strip
+      handle_signal(signal)
+    end
+    # normal
+  rescue Interrupt
+  rescue => e
+    raise e if $DEBUG
+    warn e.message
+    warn e.backtrace.join("\n")
+    exit 1
+  ensure
+    @x.stop
+  end
+end
+
+def setup(queue)
+  ll = Loader.new
+  ll.configure(queue)
+  ll.setup(queue)
+end
+
+def consume(queues, queue, monitor_all_queues)
+  ll = Loader.new
+  ll.configure(queue)
+  ll.run(queues, queue, monitor_all_queues)
+end
+
+# We assign one queue to each sidekiq process
+def run(number_of_processes, total_queues)
+    read_stream, write_stream = IO.pipe
+
+    queues = []
+    (0..total_queues-1).each do |idx|
+      queues.push("queue:q#{idx}")
+    end
+
+    Sidekiq.logger.info("Queues are: #{queues}")
+
+    # Produce
+    start = Time.now
+    (0..total_queues-1).each do |idx|
+      Process.fork do
+          queue_num = "q#{idx}"
+          setup(queue_num)
+      end
+    end
+
+    queue_sz = $iterations * $elements * total_queues
+    Process.waitall
+
+    ending = Time.now - start
+    #Sidekiq.logger.info("Pushed #{queue_sz} in #{ending} secs")
+
+    # Consume
+    (0..number_of_processes-1).each do |idx|
+      Process.fork do
+          # First process only consumes from it's own queue but monitors all queues.
+          # It works as a synchronization point. Once all processes finish
+          # (that is, when all queues are emptied) it prints the the stats.
+          if idx == 0
+            queue = "q#{idx}"
+            consume(queues, queue, true)
+          else
+            queue = "q#{idx % total_queues}"
+            consume(queues, queue, false)
+          end
+      end
+    end
+
+    Process.waitall
+    write_stream.close
+    results = read_stream.read
+    read_stream.close
+end
+
+$total_processes = ENV["PROCESSES"] ? Integer(ENV["PROCESSES"]) : 8;
+$total_queues = ENV["QUEUES"] ? Integer(ENV["QUEUES"]) : 8;
+
+run($total_processes, $total_queues)

data/lib/sidekiq/api.rb

@@ -4,7 +4,6 @@ require "sidekiq"
 
 require "zlib"
 require "set"
-require "base64"
 
 require "sidekiq/metrics/query"
 
@@ -491,8 +490,8 @@ module Sidekiq
     end
 
     def uncompress_backtrace(backtrace)
-      decoded = Base64.decode64(backtrace)
-      uncompressed = Zlib::Inflate.inflate(decoded)
+      strict_base64_decoded = backtrace.unpack1("m0")
+      uncompressed = Zlib::Inflate.inflate(strict_base64_decoded)
       Sidekiq.load_json(uncompressed)
     end
   end
@@ -1110,11 +1109,11 @@ module Sidekiq
 
       procs.zip(all_works).each do |key, workers|
         workers.each_pair do |tid, json|
-          results << [key, tid, Sidekiq.load_json(json)] unless json.empty?
+          results << [key, tid, Sidekiq::Work.new(key, tid, Sidekiq.load_json(json))] unless json.empty?
         end
       end
 
-      results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
+      results.sort_by { |(_, _, hsh)| hsh.raw("run_at") }.each(&block)
     end
 
     # Note that #size is only as accurate as Sidekiq's heartbeat,
@@ -1138,6 +1137,59 @@ module Sidekiq
       end
     end
   end
+
+  # Sidekiq::Work represents a job which is currently executing.
+  class Work
+    attr_reader :process_id
+    attr_reader :thread_id
+
+    def initialize(pid, tid, hsh)
+      @process_id = pid
+      @thread_id = tid
+      @hsh = hsh
+      @job = nil
+    end
+
+    def queue
+      @hsh["queue"]
+    end
+
+    def run_at
+      Time.at(@hsh["run_at"])
+    end
+
+    def job
+      @job ||= Sidekiq::JobRecord.new(@hsh["payload"])
+    end
+
+    def payload
+      @hsh["payload"]
+    end
+
+    # deprecated
+    def [](key)
+      kwargs = {uplevel: 1}
+      kwargs[:category] = :deprecated if RUBY_VERSION > "3.0" # TODO
+      warn("Direct access to `Sidekiq::Work` attributes is deprecated, please use `#payload`, `#queue`, `#run_at` or `#job` instead", **kwargs)
+
+      @hsh[key]
+    end
+
+    # :nodoc:
+    # @api private
+    def raw(name)
+      @hsh[name]
+    end
+
+    def method_missing(*all)
+      @hsh.send(*all)
+    end
+
+    def respond_to_missing?(name)
+      @hsh.respond_to?(name)
+    end
+  end
+
   # Since "worker" is a nebulous term, we've deprecated the use of this class name.
   # Is "worker" a process, a type of job, a thread? Undefined!
   # WorkSet better describes the data.

data/lib/sidekiq/job_retry.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "zlib"
-require "base64"
 require "sidekiq/component"
 
 module Sidekiq
@@ -295,7 +294,7 @@ module Sidekiq
     def compress_backtrace(backtrace)
       serialized = Sidekiq.dump_json(backtrace)
       compressed = Zlib::Deflate.deflate(serialized)
-      Base64.encode64(compressed)
+      [compressed].pack("m0") # Base64.strict_encode64
     end
   end
 end

data/lib/sidekiq/metrics/query.rb

@@ -119,6 +119,7 @@ module Sidekiq
 
         def total_avg(metric = "ms")
           completed = totals["p"] - totals["f"]
+          return 0 if completed.zero?
           totals[metric].to_f / completed
         end
 

data/lib/sidekiq/metrics/tracking.rb

@@ -103,12 +103,16 @@ module Sidekiq
       def reset
         @lock.synchronize {
           array = [@totals, @jobs, @grams]
-          @totals = Hash.new(0)
-          @jobs = Hash.new(0)
-          @grams = Hash.new { |hash, key| hash[key] = Histogram.new(key) }
+          reset_instance_variables
           array
         }
       end
+
+      def reset_instance_variables
+        @totals = Hash.new(0)
+        @jobs = Hash.new(0)
+        @grams = Hash.new { |hash, key| hash[key] = Histogram.new(key) }
+      end
     end
 
     class Middleware

data/lib/sidekiq/rails.rb

@@ -56,10 +56,10 @@ module Sidekiq
         # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
         # it will appear in the Sidekiq console with all of the job context.
         unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
-          if ::Rails::VERSION::STRING < "7.1"
-            ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
-          else
+          if ::Rails.logger.respond_to?(:broadcast_to)
             ::Rails.logger.broadcast_to(config.logger)
+          else
+            ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
           end
         end
       end

data/lib/sidekiq/redis_connection.rb

@@ -39,9 +39,8 @@ module Sidekiq
           uri.password = redacted
           scrubbed_options[:url] = uri.to_s
         end
-        if scrubbed_options[:password]
-          scrubbed_options[:password] = redacted
-        end
+        scrubbed_options[:password] = redacted if scrubbed_options[:password]
+        scrubbed_options[:sentinel_password] = redacted if scrubbed_options[:sentinel_password]
         scrubbed_options[:sentinels]&.each do |sentinel|
           sentinel[:password] = redacted if sentinel[:password]
         end

data/lib/sidekiq/testing.rb

@@ -278,7 +278,7 @@ module Sidekiq
       def perform_one
         raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
         next_job = jobs.first
-        Queues.delete_for(next_job["jid"], queue, to_s)
+        Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
         process_job(next_job)
       end
 

data/lib/sidekiq/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "7.2.0"
+  VERSION = "7.2.1"
   MAJOR = 7
 end

data/lib/sidekiq/web/csrf_protection.rb

@@ -27,7 +27,6 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 require "securerandom"
-require "base64"
 require "rack/request"
 
 module Sidekiq
@@ -143,7 +142,7 @@ module Sidekiq
         one_time_pad = SecureRandom.random_bytes(token.length)
         encrypted_token = xor_byte_strings(one_time_pad, token)
         masked_token = one_time_pad + encrypted_token
-        Base64.urlsafe_encode64(masked_token)
+        encode_token(masked_token)
       end
 
       # Essentially the inverse of +mask_token+.
@@ -168,8 +167,12 @@ module Sidekiq
         ::Rack::Utils.secure_compare(token.to_s, decode_token(local).to_s)
       end
 
+      def encode_token(token)
+        [token].pack("m0").tr("+/", "-_")
+      end
+
       def decode_token(token)
-        Base64.urlsafe_decode64(token)
+        token.tr("-_", "+/").unpack1("m0")
       end
 
       def xor_byte_strings(s1, s2)

data/sidekiq.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |gem|
     "rubygems_mfa_required" => "true"
   }
 
-  gem.add_dependency "redis-client", ">= 0.14.0"
+  gem.add_dependency "redis-client", ">= 0.19.0"
   gem.add_dependency "connection_pool", ">= 2.3.0"
   gem.add_dependency "rack", ">= 2.2.4"
   gem.add_dependency "concurrent-ruby", "< 2"

data/web/views/busy.erb

@@ -125,14 +125,14 @@
       <th><%= t('Arguments') %></th>
       <th><%= t('Started') %></th>
     </thead>
-    <% @workset.each do |process, thread, msg| %>
-      <% job = Sidekiq::JobRecord.new(msg['payload']) %>
+    <% @workset.each do |process, thread, work| %>
+      <% job = work.job %>
       <tr>
         <td><%= process %></td>
         <td><%= thread %></td>
         <td><%= job.jid %></td>
         <td>
-          <a href="<%= root_path %>queues/<%= msg['queue'] %>"><%= msg['queue'] %></a>
+          <a href="<%= root_path %>queues/<%= work.queue %>"><%= work.queue %></a>
         </td>
         <td>
           <%= job.display_class %>
@@ -141,7 +141,7 @@
         <td>
           <div class="args"><%= display_args(job.display_args) %></div>
         </td>
-        <td><%= relative_time(Time.at(msg['run_at'])) %></td>
+        <td><%= relative_time(work.run_at) %></td>
       </tr>
     <% end %>
   </table>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 7.2.0
+  version: 7.2.1
 platform: ruby
 authors:
 - Mike Perham
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-31 00:00:00.000000000 Z
+date: 2024-01-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-client
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: 0.19.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: 0.19.0
 - !ruby/object:Gem::Dependency
   name: connection_pool
   requirement: !ruby/object:Gem::Requirement
@@ -78,6 +78,7 @@ files:
 - Changes.md
 - LICENSE.txt
 - README.md
+- bin/multi_queue_bench
 - bin/sidekiq
 - bin/sidekiqload
 - bin/sidekiqmon