sidekiq 7.3.4 → 7.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e334af4f9c2d4bbf79ec6b72e2fe344d07f4660534ca4869ccdfa95274a0d58
-  data.tar.gz: 1cbea7241abbe7a85800e5e9df75491695d1c10a8af66f4dd7b84e174c23b7ea
+  metadata.gz: 6c9bd0a03b27535a34a0525397e84c429a9332e6754dba43c088558ab8f362e9
+  data.tar.gz: '08fee2db2d83996d1df6f78e7eb9a226cb98d951cba336dd422771f9127cf95a'
 SHA512:
-  metadata.gz: 8dd6fc27c641bf6ded12b52784dfdc99691d810d40a160f2a5dd19e0a49bf1530a6314b48bc248b6ec66127fa108c2186b00d4affe69505040306e0aca5a08eb
-  data.tar.gz: 7a108824b94fd1e80c62e510cc5efc9aba2b62b8af83c6d4fe4c7cca19eb5a4a17a46abe32b74f0ab2a1cb3187f5688a0058f8f63ef0b1121cb44faae50e1546
+  metadata.gz: f4876eb7594d0c87e4dba08a752dbcb65bbf45ae8213fb9b9ee6a1e12d7a48750c161ffe632f3b13eacfed73de3c3c73a9dc06a2548ddf7e5f6dde0f7de161bd
+  data.tar.gz: a0f485adef4ba6751114daf0d2c1d509966d530bc620e00ebf17a9c244f1eb0821ae5e8214d88d3652106d932cae7acfd2b66874647acc8c33fe6ec2e3b9d691

data/Changes.md

@@ -2,6 +2,22 @@
 
 [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
 
+7.3.6
+----------
+
+- Forward compatibility fixes for Ruby 3.4
+- Filtering in the Web UI now works via GET so you can bookmark a filtered view. [#6497]
+
+7.3.5
+----------
+
+- Reimplement `retry_all` and `kill_all` API methods to use ZPOPMIN,
+  approximately 30-60% faster. [#6481]
+- Add preload testing binary at `examples/testing/sidekiq_boot` to verify your Rails app boots correctly with Sidekiq Enterprise's app preloading.
+- Fix circular require with ActiveJob adapter [#6477]
+- Fix potential race condition leading to incorrect serialized values for CurrentAttributes [#6475]
+- Restore missing elapsed time when default job logging is disabled
+
 7.3.4
 ----------
 

data/bin/sidekiqload

@@ -50,7 +50,7 @@ if ENV["AJ"]
   ActiveJob::Base.logger.level = Logger::WARN
 
   class LoadJob < ActiveJob::Base
-    def perform(idx, ts = nil)
+    def perform(string, idx, hash, ts = nil)
       puts(Time.now.to_f - ts) if !ts.nil?
     end
   end
@@ -58,12 +58,21 @@ end
 
 class LoadWorker
   include Sidekiq::Job
+  $count = 0
+  $lock = Mutex.new
+ 
   sidekiq_options retry: 1
   sidekiq_retry_in do |x|
     1
   end
 
-  def perform(idx, ts = nil)
+  def perform(string, idx, hash, ts = nil)
+    $lock.synchronize do
+      $count += 1
+      if $count % 100_000 == 0
+        logger.warn("#{Time.now} Done #{$count}")
+      end
+    end
     puts(Time.now.to_f - ts) if !ts.nil?
     # raise idx.to_s if idx % 100 == 1
   end
@@ -133,13 +142,13 @@ class Loader
     start = Time.now
     if ENV["AJ"]
       @iter.times do
-        @count.times do |idx|
-          LoadJob.perform_later(idx)
-        end
+        ActiveJob.perform_all_later(@count.times.map do |idx|
+          LoadJob.new("mike", idx, {mike: "bob"})
+        end)
       end
     else
       @iter.times do
-        arr = Array.new(@count) { |idx| [idx] }
+        arr = Array.new(@count) { |idx| ["string", idx, {"mike" => "bob"}] }
         Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
       end
     end
@@ -163,13 +172,13 @@ class Loader
           Sidekiq.logger.error("Now here's the latency for three jobs")
 
           if ENV["AJ"]
-            LoadJob.perform_later(1, Time.now.to_f)
-            LoadJob.perform_later(2, Time.now.to_f)
-            LoadJob.perform_later(3, Time.now.to_f)
+            LoadJob.perform_later("", 1, {}, Time.now.to_f)
+            LoadJob.perform_later("", 2, {}, Time.now.to_f)
+            LoadJob.perform_later("", 3, {}, Time.now.to_f)
           else
-            LoadWorker.perform_async(1, Time.now.to_f)
-            LoadWorker.perform_async(2, Time.now.to_f)
-            LoadWorker.perform_async(3, Time.now.to_f)
+            LoadWorker.perform_async("", 1, {}, Time.now.to_f)
+            LoadWorker.perform_async("", 2, {}, Time.now.to_f)
+            LoadWorker.perform_async("", 3, {}, Time.now.to_f)
           end
 
           sleep 0.1

data/lib/active_job/queue_adapters/sidekiq_adapter.rb

@@ -16,7 +16,7 @@ end
 module ActiveJob
   module QueueAdapters
     # Explicitly remove the implementation existing in older rails'.
-    remove_const(:SidekiqAdapter) if defined?(:SidekiqAdapter)
+    remove_const(:SidekiqAdapter) if const_defined?(:SidekiqAdapter)
 
     # Sidekiq adapter for Active Job
     #

data/lib/sidekiq/api.rb

@@ -668,6 +668,41 @@ module Sidekiq
       end
     end
 
+    def pop_each
+      Sidekiq.redis do |c|
+        size.times do
+          data, score = c.zpopmin(name, 1)&.first
+          break unless data
+          yield data, score
+        end
+      end
+    end
+
+    def retry_all
+      c = Sidekiq::Client.new
+      pop_each do |msg, _|
+        job = Sidekiq.load_json(msg)
+        # Manual retries should not count against the retry limit.
+        job["retry_count"] -= 1 if job["retry_count"]
+        c.push(job)
+      end
+    end
+
+    # Move all jobs from this Set to the Dead Set.
+    # See DeadSet#kill
+    def kill_all(notify_failure: false, ex: nil)
+      ds = DeadSet.new
+      opts = {notify_failure: notify_failure, ex: ex, trim: false}
+
+      begin
+        pop_each do |msg, _|
+          ds.kill(msg, opts)
+        end
+      ensure
+        ds.trim
+      end
+    end
+
     def each
       initial_size = @_size
       offset_size = 0
@@ -765,10 +800,6 @@ module Sidekiq
 
   ##
   # The set of scheduled jobs within Sidekiq.
-  # Based on this, you can search/filter for jobs.  Here's an
-  # example where I'm selecting jobs based on some complex logic
-  # and deleting them from the scheduled set.
-  #
   # See the API wiki page for usage notes and examples.
   #
   class ScheduledSet < JobSet
@@ -779,26 +810,12 @@ module Sidekiq
 
   ##
   # The set of retries within Sidekiq.
-  # Based on this, you can search/filter for jobs.  Here's an
-  # example where I'm selecting all jobs of a certain type
-  # and deleting them from the retry queue.
-  #
   # See the API wiki page for usage notes and examples.
   #
   class RetrySet < JobSet
     def initialize
       super("retry")
     end
-
-    # Enqueues all jobs pending within the retry set.
-    def retry_all
-      each(&:retry) while size > 0
-    end
-
-    # Kills all jobs pending within the retry set.
-    def kill_all
-      each(&:kill) while size > 0
-    end
   end
 
   ##
@@ -811,20 +828,31 @@ module Sidekiq
       super("dead")
     end
 
+    # Trim dead jobs which are over our storage limits
+    def trim
+      hash = Sidekiq.default_configuration
+      now = Time.now.to_f
+      Sidekiq.redis do |conn|
+        conn.multi do |transaction|
+          transaction.zremrangebyscore(name, "-inf", now - hash[:dead_timeout_in_seconds])
+          transaction.zremrangebyrank(name, 0, - hash[:dead_max_jobs])
+        end
+      end
+    end
+
     # Add the given job to the Dead set.
     # @param message [String] the job data as JSON
-    # @option opts [Boolean] :notify_failure  (true) Whether death handlers should be called
+    # @option opts [Boolean] :notify_failure (true) Whether death handlers should be called
+    # @option opts [Boolean] :trim (true) Whether Sidekiq should trim the structure to keep it within configuration
     # @option opts [Exception] :ex (RuntimeError) An exception to pass to the death handlers
     def kill(message, opts = {})
       now = Time.now.to_f
       Sidekiq.redis do |conn|
-        conn.multi do |transaction|
-          transaction.zadd(name, now.to_s, message)
-          transaction.zremrangebyscore(name, "-inf", now - Sidekiq::Config::DEFAULTS[:dead_timeout_in_seconds])
-          transaction.zremrangebyrank(name, 0, - Sidekiq::Config::DEFAULTS[:dead_max_jobs])
-        end
+        conn.zadd(name, now.to_s, message)
       end
 
+      trim if opts[:trim] != false
+
       if opts[:notify_failure] != false
         job = Sidekiq.load_json(message)
         if opts[:ex]
@@ -839,11 +867,6 @@ module Sidekiq
       end
       true
     end
-
-    # Enqueue all dead jobs
-    def retry_all
-      each(&:retry) while size > 0
-    end
   end
 
   ##

data/lib/sidekiq/cli.rb

@@ -101,7 +101,7 @@ module Sidekiq # :nodoc:
       # Touch middleware so it isn't lazy loaded by multiple threads, #3043
       @config.server_middleware
 
-      ::Process.warmup if warmup && ::Process.respond_to?(:warmup)
+      ::Process.warmup if warmup && ::Process.respond_to?(:warmup) && ENV["RUBY_DISABLE_WARMUP"] != "1"
 
       # Before this point, the process is initializing with just the main thread.
       # Starting here the process will now have multiple threads running.

data/lib/sidekiq/job_logger.rb

@@ -5,31 +5,21 @@ module Sidekiq
     def initialize(config)
       @config = config
       @logger = @config.logger
-    end
-
-    # If true we won't do any job logging out of the box.
-    # The user is responsible for any logging.
-    def skip_default_logging?
-      @config[:skip_default_job_logging]
+      @skip = !!@config[:skip_default_job_logging]
     end
 
     def call(item, queue)
-      return yield if skip_default_logging?
-
-      begin
-        start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-        @logger.info("start")
+      start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      @logger.info { "start" } unless @skip
 
-        yield
-
-        Sidekiq::Context.add(:elapsed, elapsed(start))
-        @logger.info("done")
-      rescue Exception
-        Sidekiq::Context.add(:elapsed, elapsed(start))
-        @logger.info("fail")
+      yield
 
-        raise
-      end
+      Sidekiq::Context.add(:elapsed, elapsed(start))
+      @logger.info { "done" } unless @skip
+    rescue Exception
+      Sidekiq::Context.add(:elapsed, elapsed(start))
+      @logger.info { "fail" } unless @skip
+      raise
     end
 
     def prepare(job_hash, &block)

data/lib/sidekiq/middleware/current_attributes.rb

@@ -33,11 +33,26 @@ module Sidekiq
             attrs = strklass.constantize.attributes
             # Retries can push the job N times, we don't
             # want retries to reset cattr. #5692, #5090
-            job[key] = attrs if attrs.any?
+            if attrs.any?
+              # Older rails has a bug that `CurrentAttributes#attributes` always returns
+              # the same hash instance. We need to dup it to avoid being accidentally mutated.
+              job[key] = if returns_same_object?
+                attrs.dup
+              else
+                attrs
+              end
+            end
           end
         end
         yield
       end
+
+      private
+
+      def returns_same_object?
+        ActiveSupport::VERSION::MAJOR < 8 ||
+          (ActiveSupport::VERSION::MAJOR == 8 && ActiveSupport::VERSION::MINOR == 0)
+      end
     end
 
     class Load

data/lib/sidekiq/rails.rb

@@ -3,12 +3,6 @@
 require "sidekiq/job"
 require "rails"
 
-begin
-  require "active_job"
-  require "active_job/queue_adapters/sidekiq_adapter"
-rescue LoadError
-end
-
 module Sidekiq
   class Rails < ::Rails::Engine
     class Reloader
@@ -45,6 +39,8 @@ module Sidekiq
     #   end
     initializer "sidekiq.active_job_integration" do
       ActiveSupport.on_load(:active_job) do
+        require "active_job/queue_adapters/sidekiq_adapter"
+
         include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
       end
     end

data/lib/sidekiq/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "7.3.4"
+  VERSION = "7.3.6"
   MAJOR = 7
 
   def self.gem_version

data/lib/sidekiq/web/application.rb

@@ -67,11 +67,15 @@ module Sidekiq
     end
 
     get "/metrics" do
+      x = params[:substr]
+      class_filter = (x.nil? || x == "") ? nil : Regexp.new(Regexp.escape(x), Regexp::IGNORECASE)
+
       q = Sidekiq::Metrics::Query.new
       @period = h((params[:period] || "")[0..1])
       @periods = METRICS_PERIODS
       minutes = @periods.fetch(@period, @periods.values.first)
-      @query_result = q.top_jobs(minutes: minutes)
+      @query_result = q.top_jobs(minutes: minutes, class_filter: class_filter)
+
       erb(:metrics)
     end
 
@@ -153,9 +157,15 @@ module Sidekiq
     end
 
     get "/morgue" do
-      @count = (params["count"] || 25).to_i
-      (@current_page, @total_size, @dead) = page("dead", params["page"], @count, reverse: true)
-      @dead = @dead.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      x = params[:substr]
+
+      if x && x != ""
+        @dead = search(Sidekiq::DeadSet.new, x)
+      else
+        @count = (params["count"] || 25).to_i
+        (@current_page, @total_size, @dead) = page("dead", params["page"], @count, reverse: true)
+        @dead = @dead.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      end
 
       erb(:morgue)
     end
@@ -207,9 +217,15 @@ module Sidekiq
     end
 
     get "/retries" do
-      @count = (params["count"] || 25).to_i
-      (@current_page, @total_size, @retries) = page("retry", params["page"], @count)
-      @retries = @retries.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      x = params[:substr]
+
+      if x && x != ""
+        @retries = search(Sidekiq::RetrySet.new, x)
+      else
+        @count = (params["count"] || 25).to_i
+        (@current_page, @total_size, @retries) = page("retry", params["page"], @count)
+        @retries = @retries.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      end
 
       erb(:retries)
     end
@@ -262,9 +278,15 @@ module Sidekiq
     end
 
     get "/scheduled" do
-      @count = (params["count"] || 25).to_i
-      (@current_page, @total_size, @scheduled) = page("schedule", params["page"], @count)
-      @scheduled = @scheduled.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      x = params[:substr]
+
+      if x && x != ""
+        @scheduled = search(Sidekiq::ScheduledSet.new, x)
+      else
+        @count = (params["count"] || 25).to_i
+        (@current_page, @total_size, @scheduled) = page("schedule", params["page"], @count)
+        @scheduled = @scheduled.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      end
 
       erb(:scheduled)
     end
@@ -328,46 +350,6 @@ module Sidekiq
       json Sidekiq::Stats.new.queues
     end
 
-    ########
-    # Filtering
-
-    route :get, :post, "/filter/metrics" do
-      x = params[:substr]
-      return redirect "#{root_path}metrics" unless x && x != ""
-
-      q = Sidekiq::Metrics::Query.new
-      @period = h((params[:period] || "")[0..1])
-      @periods = METRICS_PERIODS
-      minutes = @periods.fetch(@period, @periods.values.first)
-      @query_result = q.top_jobs(minutes: minutes, class_filter: Regexp.new(Regexp.escape(x), Regexp::IGNORECASE))
-
-      erb :metrics
-    end
-
-    route :get, :post, "/filter/retries" do
-      x = params[:substr]
-      return redirect "#{root_path}retries" unless x && x != ""
-
-      @retries = search(Sidekiq::RetrySet.new, params[:substr])
-      erb :retries
-    end
-
-    route :get, :post, "/filter/scheduled" do
-      x = params[:substr]
-      return redirect "#{root_path}scheduled" unless x && x != ""
-
-      @scheduled = search(Sidekiq::ScheduledSet.new, params[:substr])
-      erb :scheduled
-    end
-
-    route :get, :post, "/filter/dead" do
-      x = params[:substr]
-      return redirect "#{root_path}morgue" unless x && x != ""
-
-      @dead = search(Sidekiq::DeadSet.new, params[:substr])
-      erb :morgue
-    end
-
     post "/change_locale" do
       locale = params["locale"]
 

data/web/assets/stylesheets/application.css

@@ -650,6 +650,11 @@ div.interval-slider input {
   .redis-url {
     max-width: 160px;
   }
+
+  .navbar-fixed-bottom .nav {
+    margin-left: -15px;
+    margin-right: -15px;
+  }
 }
 
 @media (min-width: 992px) {

data/web/locales/it.yml

@@ -6,44 +6,60 @@ it:
   AreYouSureDeleteJob: Sei sicuro di voler cancellare questo lavoro?
   AreYouSureDeleteQueue: Sei sicuro di voler cancellare la coda %{queue}?
   Arguments: Argomenti
+  BackToApp: Torna all'App
   Busy: Occupato
   Class: Classe
   Connections: Connessioni
+  CreatedAt: Creato il
   CurrentMessagesInQueue: Messaggi in <span class='title'>%{queue}</span>
   Dashboard: Dashboard
   Dead: Arrestato
   DeadJobs: Lavori arrestati
   Delete: Cancella
   DeleteAll: Cancella tutti
+  Deploy: Distribuire
   Enqueued: In coda
   Error: Errore
   ErrorBacktrace: Backtrace dell'errore
   ErrorClass: Classe dell'errore
   ErrorMessage: Messaggio di errore
+  ExecutionTime: Tempo di esecuzione
   Extras: Extra
   Failed: Fallito
   Failures: Fallimenti
+  Failure: Fallimento
   GoBack: ← Indietro
   History: Storia
   Job: Lavoro
   Jobs: Lavori
   Kill: Uccidere
+  KillAll: Uccidere tutti
   LastRetry: Ultimo tentativo
+  Latency: Latenza
   LivePoll: Live poll
   MemoryUsage: Memoria utilizzata
+  Name: Nome
   Namespace: Namespace
   NextRetry: Prossimo tentativo
   NoDeadJobsFound: Non ci sono lavori arrestati
   NoRetriesFound: Non sono stati trovati nuovi tentativi
   NoScheduledFound: Non ci sono lavori pianificati
+  NotYetEnqueued: Non ancora in coda
   OneMonth: 1 mese
   OneWeek: 1 settimana
   OriginallyFailed: Primo fallimento
+  Pause: Metti in pausa
+  Paused: In pausa
   PeakMemoryUsage: Memoria utilizzata (max.)
+  Plugins: Plugins
+  PollingInterval: Intervallo di polling
+  Process: Processo
   Processed: Processato
   Processes: Processi
   Queue: Coda
   Queues: Code
+  Quiet: Silenzia
+  QuietAll: Silenzia Tutti
   Realtime: Tempo reale
   Retries: Nuovi tentativi
   RetryAll: Riprova tutti
@@ -51,19 +67,34 @@ it:
   RetryNow: Riprova
   Scheduled: Pianificato
   ScheduledJobs: Lavori pianificati
+  Seconds: Secondi
   ShowAll: Mostra tutti
   SixMonths: 6 mesi
   Size: Dimensione
   Started: Iniziato
   Status: Stato
+  Stop: Ferma
+  StopAll: Ferma Tutti
   StopPolling: Ferma il polling
+  Success: Successo
+  Summary: Riepilogo
   Thread: Thread
-  Threads: Thread
+  Threads: Threads
   ThreeMonths: 3 mesi
   Time: Ora
+  Unpause: Riattiva
   Uptime: Uptime (giorni)
+  Utilization: Utilizzo
   Version: Versione
   When: Quando
   Worker: Lavoratore
   active: attivo
   idle: inattivo
+  Metrics: Metriche
+  NoDataFound: Nessun dato trovato
+  TotalExecutionTime: Tempo totale di esecuzione
+  AvgExecutionTime: Tempo medio di esecuzione
+  Context: Contesto
+  NoJobMetricsFound: Metriche recenti di lavoro non trovate
+  Filter: Filtro
+  AnyJobContent: Qualsiasi contenuto di lavoro

data/web/views/filtering.erb

@@ -1,6 +1,5 @@
 <div>
-  <form method="post" class="form-inline" action='<%= root_path %>filter/<%= which %>'>
-    <%= csrf_tag %>
+  <form method="get" class="form-inline" action='<%= root_path %><%= which %>'>
     <label for="substr"><%= t('Filter') %></label>
     <input class="search form-control" type="search" name="substr" value="<%= h params[:substr] %>" placeholder="<%= t('AnyJobContent') %>"/>
   </form>

data/web/views/metrics.erb

@@ -9,8 +9,7 @@
   </div>
 
   <div>
-    <form id="metrics-form" class="form-inline" action="<%= root_path %>filter/metrics" method="post">
-      <%= csrf_tag %>
+    <form id="metrics-form" class="form-inline" action="<%= root_path %>metrics" method="get">
       <label for="substr"><%= t('Filter') %></label>
       <input id="class-filter" class="form-control" type="text" name="substr" placeholder="<%= t('Name') %>" value="<%= h params[:substr] %>">
       <select id="period-selector" class="form-control" name="period">

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 7.3.4
+  version: 7.3.6
 platform: ruby
 authors:
 - Mike Perham
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-23 00:00:00.000000000 Z
+date: 2024-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-client
@@ -232,7 +232,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.5.16
 signing_key:
 specification_version: 4
 summary: Simple, efficient background processing for Ruby