sidekiq 7.3.0 → 7.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34279eae9b8159c48dd31c7b088341136d8b785e7b853389e29e179443af1577
-  data.tar.gz: 520c6f705995df5c8e50e4e607956bcd8fe78d332e9a33fdf7a4ff25b7534f60
+  metadata.gz: dccc402e8f4e309a47a2adb62418fa29989012e8c94f34aa11200b098dc375f2
+  data.tar.gz: 71b64a582d5971b045d60bae38dc26da265b60ff9a099ca3b2f9c7df9035e805
 SHA512:
-  metadata.gz: 939c535c352b1c9390e7675d75cfc3728685282187c0edd8475373c6bad1ee67391477a46e26ea3de06724e8cf2570ef759862eb37234c1f9d53f79adafd5947
-  data.tar.gz: 1a127bf11d514789f6ec808f964d47e9d459687a5b2d91ebe468d6a59a28c5f9dbf01203e1aee583311c776ca9caa41831d74a15557adf5bb0bd71cce27cffdd
+  metadata.gz: d9148b613a222ca9617ceebe25bb04a5d086edacdafad4a426b6232662d3b583db462dc5e0491297f7859037c5166bcb541bddcd3949f6d3b5a1c2e3fc572b65
+  data.tar.gz: 93a797cefd1a68adb236c7538c28571467c328d6965af0bc5ce505a1391d6f5b6e2ae2c87b42110e837dcfa8d788aff55309806bbb1bcd181e205c3bc66adc29

data/Changes.md

@@ -2,6 +2,87 @@
 
 [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
 
+7.3.9
+----------
+
+- Only require activejob if necessary [#6584]
+- Fix iterable job cancellation [#6589]
+- Web UI accessibility improvements [#6604]
+
+7.3.8
+----------
+
+- Fix dead tag links [#6554]
+- Massive Web UI performance improvement, some pages up to 15x faster [#6555]
+
+7.3.7
+----------
+
+- Backport `Sidekiq::Web.configure` for compatibility with 8.0 [#6532]
+- Backport `url_params(key)` and `route_params(key)` for compatibility with 8.0 [#6532]
+- Various fixes for UI filtering [#6508]
+- Tune `inspect` for internal S::Components to keep size managable [#6553]
+
+7.3.6
+----------
+
+- Forward compatibility fixes for Ruby 3.4
+- Filtering in the Web UI now works via GET so you can bookmark a filtered view. [#6497]
+
+7.3.5
+----------
+
+- Reimplement `retry_all` and `kill_all` API methods to use ZPOPMIN,
+  approximately 30-60% faster. [#6481]
+- Add preload testing binary at `examples/testing/sidekiq_boot` to verify your Rails app boots correctly with Sidekiq Enterprise's app preloading.
+- Fix circular require with ActiveJob adapter [#6477]
+- Fix potential race condition leading to incorrect serialized values for CurrentAttributes [#6475]
+- Restore missing elapsed time when default job logging is disabled
+
+7.3.4
+----------
+
+- Fix FrozenError when starting Sidekiq [#6470]
+
+7.3.3
+----------
+
+- Freeze global configuration once boot is complete, to avoid configuration race conditions [#6466, #6465]
+- Sidekiq now warns if a job iteration takes longer than the `-t` timeout setting (defaults to 25 seconds)
+- Iteration callbacks now have easy access to job arguments via the `arguments` method:
+```ruby
+def on_stop
+  p arguments # => `[123, "string", {"key" => "value"}]`
+  id, str, hash = arguments
+end
+```
+- Iterable jobs can be cancelled via `Sidekiq::Client#cancel!`:
+```ruby
+c = Sidekiq::Client.new
+jid = c.push("class" => SomeJob, "args" => [123])
+c.cancel!(jid) # => true
+```
+- Take over support for ActiveJob's `:sidekiq` adapter [#6430, fatkodima]
+- Ensure CurrentAttributes are in scope when creating batch callbacks [#6455]
+- Add `Sidekiq.gem_version` API.
+- Update Ukranian translations
+
+7.3.2
+----------
+
+- Adjust ActiveRecord batch iteration to restart an interrupted batch from the beginning.
+  Each batch should be processed as a single transaction in order to be idempotent. [#6405]
+- Fix typo in Sidekiq::DeadSet#kill [#6397]
+- Fix CSS issue with bottom bar in Web UI [#6414]
+
+7.3.1
+----------
+
+- Don't count job interruptions as failures in metrics [#6386]
+- Add frozen string literal to a number of .rb files.
+- Fix frozen string error with style_tag and script_tag [#6371]
+- Fix an error on Ruby 2.7 because of usage of `Hash#except` [#6376]
+
 7.3.0
 ----------
 

data/bin/sidekiqload

@@ -50,7 +50,7 @@ if ENV["AJ"]
   ActiveJob::Base.logger.level = Logger::WARN
 
   class LoadJob < ActiveJob::Base
-    def perform(idx, ts = nil)
+    def perform(string, idx, hash, ts = nil)
       puts(Time.now.to_f - ts) if !ts.nil?
     end
   end
@@ -58,12 +58,21 @@ end
 
 class LoadWorker
   include Sidekiq::Job
+  $count = 0
+  $lock = Mutex.new
+ 
   sidekiq_options retry: 1
   sidekiq_retry_in do |x|
     1
   end
 
-  def perform(idx, ts = nil)
+  def perform(string, idx, hash, ts = nil)
+    $lock.synchronize do
+      $count += 1
+      if $count % 100_000 == 0
+        logger.warn("#{Time.now} Done #{$count}")
+      end
+    end
     puts(Time.now.to_f - ts) if !ts.nil?
     # raise idx.to_s if idx % 100 == 1
   end
@@ -133,13 +142,13 @@ class Loader
     start = Time.now
     if ENV["AJ"]
       @iter.times do
-        @count.times do |idx|
-          LoadJob.perform_later(idx)
-        end
+        ActiveJob.perform_all_later(@count.times.map do |idx|
+          LoadJob.new("mike", idx, {mike: "bob"})
+        end)
       end
     else
       @iter.times do
-        arr = Array.new(@count) { |idx| [idx] }
+        arr = Array.new(@count) { |idx| ["string", idx, {"mike" => "bob"}] }
         Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
       end
     end
@@ -163,13 +172,13 @@ class Loader
           Sidekiq.logger.error("Now here's the latency for three jobs")
 
           if ENV["AJ"]
-            LoadJob.perform_later(1, Time.now.to_f)
-            LoadJob.perform_later(2, Time.now.to_f)
-            LoadJob.perform_later(3, Time.now.to_f)
+            LoadJob.perform_later("", 1, {}, Time.now.to_f)
+            LoadJob.perform_later("", 2, {}, Time.now.to_f)
+            LoadJob.perform_later("", 3, {}, Time.now.to_f)
           else
-            LoadWorker.perform_async(1, Time.now.to_f)
-            LoadWorker.perform_async(2, Time.now.to_f)
-            LoadWorker.perform_async(3, Time.now.to_f)
+            LoadWorker.perform_async("", 1, {}, Time.now.to_f)
+            LoadWorker.perform_async("", 2, {}, Time.now.to_f)
+            LoadWorker.perform_async("", 3, {}, Time.now.to_f)
           end
 
           sleep 0.1

data/lib/active_job/queue_adapters/sidekiq_adapter.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module QueueAdapters
+    # Explicitly remove the implementation existing in older rails'.
+    remove_const(:SidekiqAdapter) if const_defined?(:SidekiqAdapter)
+
+    # Sidekiq adapter for Active Job
+    #
+    # To use Sidekiq set the queue_adapter config to +:sidekiq+.
+    #
+    #   Rails.application.config.active_job.queue_adapter = :sidekiq
+    class SidekiqAdapter
+      # Defines whether enqueuing should happen implicitly to after commit when called
+      # from inside a transaction.
+      # @api private
+      def enqueue_after_transaction_commit?
+        true
+      end
+
+      # @api private
+      def enqueue(job)
+        job.provider_job_id = JobWrapper.set(
+          wrapped: job.class,
+          queue: job.queue_name
+        ).perform_async(job.serialize)
+      end
+
+      # @api private
+      def enqueue_at(job, timestamp)
+        job.provider_job_id = JobWrapper.set(
+          wrapped: job.class,
+          queue: job.queue_name
+        ).perform_at(timestamp, job.serialize)
+      end
+
+      # @api private
+      def enqueue_all(jobs)
+        enqueued_count = 0
+        jobs.group_by(&:class).each do |job_class, same_class_jobs|
+          same_class_jobs.group_by(&:queue_name).each do |queue, same_class_and_queue_jobs|
+            immediate_jobs, scheduled_jobs = same_class_and_queue_jobs.partition { |job| job.scheduled_at.nil? }
+
+            if immediate_jobs.any?
+              jids = Sidekiq::Client.push_bulk(
+                "class" => JobWrapper,
+                "wrapped" => job_class,
+                "queue" => queue,
+                "args" => immediate_jobs.map { |job| [job.serialize] }
+              )
+              enqueued_count += jids.compact.size
+            end
+
+            if scheduled_jobs.any?
+              jids = Sidekiq::Client.push_bulk(
+                "class" => JobWrapper,
+                "wrapped" => job_class,
+                "queue" => queue,
+                "args" => scheduled_jobs.map { |job| [job.serialize] },
+                "at" => scheduled_jobs.map { |job| job.scheduled_at&.to_f }
+              )
+              enqueued_count += jids.compact.size
+            end
+          end
+        end
+        enqueued_count
+      end
+
+      # Defines a class alias for backwards compatibility with enqueued Active Job jobs.
+      # @api private
+      class JobWrapper < Sidekiq::ActiveJob::Wrapper
+      end
+    end
+  end
+end

data/lib/generators/sidekiq/job_generator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "rails/generators/named_base"
 
 module Sidekiq

data/lib/sidekiq/api.rb

@@ -373,7 +373,7 @@ module Sidekiq
     def display_class
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
       @klass ||= self["display_class"] || begin
-        if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+        if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper" || klass == "Sidekiq::ActiveJob::Wrapper"
           job_class = @item["wrapped"] || args[0]
           if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
             # MailerClass#mailer_method
@@ -389,7 +389,7 @@ module Sidekiq
 
     def display_args
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
-      @display_args ||= if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+      @display_args ||= if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper" || klass == "Sidekiq::ActiveJob::Wrapper"
         job_args = self["wrapped"] ? deserialize_argument(args[0]["arguments"]) : []
         if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
           # remove MailerClass, mailer_method and 'deliver_now'
@@ -668,6 +668,41 @@ module Sidekiq
       end
     end
 
+    def pop_each
+      Sidekiq.redis do |c|
+        size.times do
+          data, score = c.zpopmin(name, 1)&.first
+          break unless data
+          yield data, score
+        end
+      end
+    end
+
+    def retry_all
+      c = Sidekiq::Client.new
+      pop_each do |msg, _|
+        job = Sidekiq.load_json(msg)
+        # Manual retries should not count against the retry limit.
+        job["retry_count"] -= 1 if job["retry_count"]
+        c.push(job)
+      end
+    end
+
+    # Move all jobs from this Set to the Dead Set.
+    # See DeadSet#kill
+    def kill_all(notify_failure: false, ex: nil)
+      ds = DeadSet.new
+      opts = {notify_failure: notify_failure, ex: ex, trim: false}
+
+      begin
+        pop_each do |msg, _|
+          ds.kill(msg, opts)
+        end
+      ensure
+        ds.trim
+      end
+    end
+
     def each
       initial_size = @_size
       offset_size = 0
@@ -765,10 +800,6 @@ module Sidekiq
 
   ##
   # The set of scheduled jobs within Sidekiq.
-  # Based on this, you can search/filter for jobs.  Here's an
-  # example where I'm selecting jobs based on some complex logic
-  # and deleting them from the scheduled set.
-  #
   # See the API wiki page for usage notes and examples.
   #
   class ScheduledSet < JobSet
@@ -779,26 +810,12 @@ module Sidekiq
 
   ##
   # The set of retries within Sidekiq.
-  # Based on this, you can search/filter for jobs.  Here's an
-  # example where I'm selecting all jobs of a certain type
-  # and deleting them from the retry queue.
-  #
   # See the API wiki page for usage notes and examples.
   #
   class RetrySet < JobSet
     def initialize
       super("retry")
     end
-
-    # Enqueues all jobs pending within the retry set.
-    def retry_all
-      each(&:retry) while size > 0
-    end
-
-    # Kills all jobs pending within the retry set.
-    def kill_all
-      each(&:kill) while size > 0
-    end
   end
 
   ##
@@ -811,33 +828,45 @@ module Sidekiq
       super("dead")
     end
 
+    # Trim dead jobs which are over our storage limits
+    def trim
+      hash = Sidekiq.default_configuration
+      now = Time.now.to_f
+      Sidekiq.redis do |conn|
+        conn.multi do |transaction|
+          transaction.zremrangebyscore(name, "-inf", now - hash[:dead_timeout_in_seconds])
+          transaction.zremrangebyrank(name, 0, - hash[:dead_max_jobs])
+        end
+      end
+    end
+
     # Add the given job to the Dead set.
     # @param message [String] the job data as JSON
+    # @option opts [Boolean] :notify_failure (true) Whether death handlers should be called
+    # @option opts [Boolean] :trim (true) Whether Sidekiq should trim the structure to keep it within configuration
+    # @option opts [Exception] :ex (RuntimeError) An exception to pass to the death handlers
     def kill(message, opts = {})
       now = Time.now.to_f
       Sidekiq.redis do |conn|
-        conn.multi do |transaction|
-          transaction.zadd(name, now.to_s, message)
-          transaction.zremrangebyscore(name, "-inf", now - Sidekiq::Config::DEFAULTS[:dead_timeout_in_seconds])
-          transaction.zremrangebyrank(name, 0, - Sidekiq::Config::DEFAULTS[:dead_max_jobs])
-        end
+        conn.zadd(name, now.to_s, message)
       end
 
+      trim if opts[:trim] != false
+
       if opts[:notify_failure] != false
         job = Sidekiq.load_json(message)
-        r = RuntimeError.new("Job killed by API")
-        r.set_backtrace(caller)
+        if opts[:ex]
+          ex = opts[:ex]
+        else
+          ex = RuntimeError.new("Job killed by API")
+          ex.set_backtrace(caller)
+        end
         Sidekiq.default_configuration.death_handlers.each do |handle|
-          handle.call(job, r)
+          handle.call(job, ex)
         end
       end
       true
     end
-
-    # Enqueue all dead jobs
-    def retry_all
-      each(&:retry) while size > 0
-    end
   end
 
   ##

data/lib/sidekiq/capsule.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "sidekiq/component"
 
 module Sidekiq
@@ -38,9 +40,9 @@ module Sidekiq
 
     def fetcher
       @fetcher ||= begin
-        inst = (config[:fetch_class] || Sidekiq::BasicFetch).new(self)
-        inst.setup(config[:fetch_setup]) if inst.respond_to?(:setup)
-        inst
+        instance = (config[:fetch_class] || Sidekiq::BasicFetch).new(self)
+        instance.setup(config[:fetch_setup]) if instance.respond_to?(:setup)
+        instance
       end
     end
 

data/lib/sidekiq/cli.rb

@@ -101,7 +101,7 @@ module Sidekiq # :nodoc:
       # Touch middleware so it isn't lazy loaded by multiple threads, #3043
       @config.server_middleware
 
-      ::Process.warmup if warmup && ::Process.respond_to?(:warmup)
+      ::Process.warmup if warmup && ::Process.respond_to?(:warmup) && ENV["RUBY_DISABLE_WARMUP"] != "1"
 
       # Before this point, the process is initializing with just the main thread.
       # Starting here the process will now have multiple threads running.

data/lib/sidekiq/client.rb

@@ -58,6 +58,23 @@ module Sidekiq
       end
     end
 
+    # Cancel the IterableJob with the given JID.
+    # **NB: Cancellation is asynchronous.** Iteration checks every
+    # five seconds so this will not immediately stop the given job.
+    def cancel!(jid)
+      key = "it-#{jid}"
+      _, result, _ = Sidekiq.redis do |c|
+        c.pipelined do |p|
+          p.hsetnx(key, "cancelled", Time.now.to_i)
+          p.hget(key, "cancelled")
+          p.expire(key, Sidekiq::Job::Iterable::STATE_TTL)
+          # TODO When Redis 7.2 is required
+          # p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
+        end
+      end
+      result.to_i
+    end
+
     ##
     # The main method used to push a job to Redis.  Accepts a number of options:
     #
@@ -251,7 +268,10 @@ module Sidekiq
           at = hash["at"].to_s
           # ActiveJob sets this but the job has not been enqueued yet
           hash.delete("enqueued_at")
-          [at, Sidekiq.dump_json(hash.except("at"))]
+          # TODO: Use hash.except("at") when support for Ruby 2.7 is dropped
+          hash = hash.dup
+          hash.delete("at")
+          [at, Sidekiq.dump_json(hash)]
         })
       else
         queue = payloads.first["queue"]

data/lib/sidekiq/component.rb

@@ -64,5 +64,27 @@ module Sidekiq
       end
       arr.clear if oneshot # once we've fired an event, we never fire it again
     end
+
+    # When you have a large tree of components, the `inspect` output
+    # can get out of hand, especially with lots of Sidekiq::Config
+    # references everywhere. We avoid calling `inspect` on more complex
+    # state and use `to_s` instead to keep output manageable, #6553
+    def inspect
+      "#<#{self.class.name} #{
+        instance_variables.map do |name|
+          value = instance_variable_get(name)
+          case value
+          when Proc
+            "#{name}=#{value}"
+          when Sidekiq::Config
+            "#{name}=#{value}"
+          when Sidekiq::Component
+            "#{name}=#{value}"
+          else
+            "#{name}=#{value.inspect}"
+          end
+        end.join(", ")
+      }>"
+    end
   end
 end

data/lib/sidekiq/config.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "forwardable"
 
 require "set"
@@ -59,6 +61,12 @@ module Sidekiq
     def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
     attr_reader :capsules
 
+    def inspect
+      "#<#{self.class.name} @options=#{
+        @options.except(:lifecycle_events, :reloader, :death_handlers, :error_handlers).inspect
+      }>"
+    end
+
     def to_json(*)
       Sidekiq.dump_json(@options)
     end
@@ -183,7 +191,13 @@ module Sidekiq
 
     # register global singletons which can be accessed elsewhere
     def register(name, instance)
-      @directory[name] = instance
+      # logger.debug("register[#{name}] = #{instance}")
+      # Sidekiq Enterprise lazy registers a few services so we
+      # can't lock down this hash completely.
+      hash = @directory.dup
+      hash[name] = instance
+      @directory = hash.freeze
+      instance
     end
 
     # find a singleton
@@ -191,10 +205,16 @@ module Sidekiq
       # JNDI is just a fancy name for a hash lookup
       @directory.fetch(name) do |key|
         return nil unless default_class
-        @directory[key] = default_class.new(self)
+        register(key, default_class.new(self))
       end
     end
 
+    def freeze!
+      @directory.freeze
+      @options.freeze
+      true
+    end
+
     ##
     # Death handlers are called when all retries for a job have been exhausted and
     # the job dies.  It's the notification to your application

data/lib/sidekiq/deploy.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "sidekiq/redis_connection"
 require "time"
 

data/lib/sidekiq/embedded.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "sidekiq/component"
 require "sidekiq/launcher"
 require "sidekiq/metrics/tracking"

data/lib/sidekiq/iterable_job.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "sidekiq/job/iterable"
 
 # Iterable jobs are ones which provide a sequence to process using

data/lib/sidekiq/job/interrupt_handler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sidekiq
   module Job
     class InterruptHandler

data/lib/sidekiq/job/iterable/active_record_enumerator.rb

@@ -22,7 +22,7 @@ module Sidekiq
         def batches
           Enumerator.new(-> { @relation.count }) do |yielder|
             @relation.find_in_batches(**@options, start: @cursor) do |batch|
-              yielder.yield(batch, batch.last.id)
+              yielder.yield(batch, batch.first.id)
             end
           end
         end
@@ -35,8 +35,8 @@ module Sidekiq
             options[:of] ||= options.delete(:batch_size)
 
             @relation.in_batches(**options, start: @cursor) do |relation|
-              last_record = relation.last
-              yielder.yield(relation, last_record.id)
+              first_record = relation.first
+              yielder.yield(relation, first_record.id)
             end
           end
         end