sidekiq 8.0.10 → 8.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a831ed9ff5c1a11b447110fa77cbccf00fc44d3e3888fd5b1baa23cce5228ee9
-  data.tar.gz: 0b11c1b102a8ff8313992201b519b87e1e9446b84c72f23369e3390f671680f6
+  metadata.gz: 4fb090d79e2cf2b320fdd47ee55a574a68a46b77a23db27b849ddb6f54acb00a
+  data.tar.gz: 4863ef28ecdd8ad2cd868caf2f47cc9302b8eb88f4ff9da4f2b5641beca50d88
 SHA512:
-  metadata.gz: f6139796abb98a9bc67bf6498438f52e135b85ea6a360f86b45ddafd4f46d9d5567592940005d93bab5c3700430adfb5be340e08ca4a9e5cbde529d19a9c7dcc
-  data.tar.gz: '08ac62d0c45ca1a8216833ecfb07ad8f37ca3d1a1137198f0994af7cf0b107c842741a4f8b62ea8c3f59b42aa209418f65d9bcf6b57127b6ae6e09daf9509907'
+  metadata.gz: 4ba71a2c43cec613337119cd29830339e213a8f876db4db84c62ac67890ee020ab841a4f7fe478f7f0fcd135571b885e54c9af12998ea6947ac51b543d52b42c
+  data.tar.gz: 9cabfc243aacec3c57c697b83374a5b934cc985c1e4ee86185a6faba7fe18f6540e3fa0b203c5dbfc0e22c03513f66e973be4152d8c8e12db29e97e21a94360f

data/Changes.md

@@ -2,6 +2,39 @@
 
 [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
 
+8.1.1
+----------
+
+- Add new `Sidekiq.testing!(mode)` API [#6931]
+Requiring code should not enable process-wide changes.
+```ruby
+# Old, implicit
+require "sidekiq/testing"
+# New, more explicit
+Sidekiq.testing!(:fake)
+```
+- Fix race condition with Stop button in UI [#6935]
+- Fix javascript error handler [#6893]
+
+8.1.0
+----------
+
+- `retry_for` and `retry` are now mutually exclusive [#6878, Saidbek]
+- `perform_inline` now enforces `strict_args!` [#6718, Saidbek]
+- Integrate Herb linting for ERB templates [#6760, Saidbek]
+- Remove CSRF code, use `Sec-Fetch-Site` header [#6874, deve1212]
+- Allow custom Web UI `assets_path` for CDN purposes [#6865, stanhu]
+- Upgrade to connection_pool 3.0
+- Allow idle connection reaping after N seconds.
+You can activate this **beta** feature like below.
+Feedback requested: is this feature stable and useful for you in production?
+This feature may or may not be enabled by default in Sidekiq 9.0.
+```ruby
+Sidekiq.configure_server do |cfg|
+  cfg.reap_idle_redis_connections(60)
+end
+```
+
 8.0.10
 ----------
 

data/README.md

@@ -97,6 +97,21 @@ Contributing
 
 See [the contributing guidelines](https://github.com/sidekiq/sidekiq/blob/main/.github/contributing.md).
 
+### ERB Linting with HERB
+
+This project uses [HERB](https://herb-tools.dev/) for ERB file linting and formatting. All ERB files have been renamed to use the `.html.erb` extension for better tooling support.
+
+**Local Development:**
+```bash
+# Run HERB linting
+bundle exec rake lint:herb
+# or
+bin/lint-herb
+```
+
+**CI Integration:**
+HERB linting is automatically run in CI to ensure all ERB files are properly formatted and free of parse errors.
+
 License
 -----------------
 

data/bin/lint-herb

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# HERB Linting Script
+# Run this script to lint all ERB files in the project
+# Usage: bin/lint-herb
+
+require "bundler/setup"
+
+puts "🔍 Running HERB linting on ERB files..."
+puts
+
+exec("bundle exec herb analyze web/views -n --no-log-file")

data/bin/tui

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require_relative "../lib/sidekiq/tui"
+
+Sidekiq::TUI.new.run

data/lib/active_job/queue_adapters/sidekiq_adapter.rb

@@ -61,10 +61,13 @@ begin
 
         # @api private
         def enqueue(job)
-          job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(
+          # NB: Active Job only serializes keys it recognizes. We
+          # cannot set arbitrary key/values here.
+          wrapper = Sidekiq::ActiveJob::Wrapper.set(
             wrapped: job.class,
             queue: job.queue_name
-          ).perform_async(job.serialize)
+          )
+          job.provider_job_id = wrapper.perform_async(job.serialize)
         end
 
         # @api private

data/lib/generators/sidekiq/job_generator.rb

@@ -14,7 +14,10 @@ module Sidekiq
       end
 
       def create_job_file
-        template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
+        template(
+          "job.rb.erb",
+          File.join(jobs_directory, class_path, "#{file_name}_job.rb")
+        )
       end
 
       def create_test_file
@@ -31,7 +34,8 @@ module Sidekiq
 
       def create_job_spec
         template_file = File.join(
-          "spec/sidekiq",
+          "spec",
+          jobs_directory.gsub("app/", ""),
           class_path,
           "#{file_name}_job_spec.rb"
         )
@@ -40,7 +44,8 @@ module Sidekiq
 
       def create_job_test
         template_file = File.join(
-          "test/sidekiq",
+          "test",
+          jobs_directory.gsub("app/", ""),
           class_path,
           "#{file_name}_job_test.rb"
         )
@@ -54,6 +59,13 @@ module Sidekiq
       def test_framework
         ::Rails.application.config.generators.options[:rails][:test_framework]
       end
+
+      # Can be set in an initializer or in application configuration
+      # with Rails.application.config.generators.options[:sidekiq][:jobs_directory] = "app/jobs"
+      # to change the directory that the job files are generated in to.
+      def jobs_directory
+        ::Rails.application.config.generators.options[:sidekiq].fetch(:jobs_directory, "app/sidekiq")
+      end
     end
   end
 end

data/lib/sidekiq/api.rb

@@ -17,12 +17,37 @@ require "sidekiq/metrics/query"
 #
 
 module Sidekiq
+  module ApiUtils
+    # @api private
+    # Calculate the latency in seconds for a job based on its enqueued timestamp
+    # @param job [Hash] the job hash
+    # @return [Float] latency in seconds
+    def calculate_latency(job)
+      timestamp = job["enqueued_at"] || job["created_at"]
+      return 0.0 unless timestamp
+
+      if timestamp.is_a?(Float)
+        # old format
+        Time.now.to_f - timestamp
+      else
+        now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
+        (now - timestamp) / 1000.0
+      end
+    end
+  end
+
   # Retrieve runtime statistics from Redis regarding
   # this Sidekiq cluster.
   #
   #   stat = Sidekiq::Stats.new
   #   stat.processed
   class Stats
+    QueueSummary = Data.define(:name, :size, :latency, :paused) do
+      alias_method :paused?, :paused
+    end
+
+    include ApiUtils
+
     def initialize
       fetch_stats_fast!
     end
@@ -63,6 +88,7 @@ module Sidekiq
       stat :default_queue_latency
     end
 
+    # @return [Hash{String => Integer}] a hash of queue names to their lengths
     def queues
       Sidekiq.redis do |conn|
         queues = conn.sscan("queues").to_a
@@ -78,6 +104,41 @@ module Sidekiq
       end
     end
 
+    # More detailed information about each queue: name, size, latency, paused status
+    # @return [Array<QueueSummary>]
+    def queue_summaries
+      Sidekiq.redis do |conn|
+        queues = conn.sscan("queues").to_a
+        return [] if queues.empty?
+
+        results = conn.pipelined { |pipeline|
+          queues.each do |queue|
+            pipeline.llen("queue:#{queue}")
+            pipeline.lindex("queue:#{queue}", -1)
+            pipeline.sismember("paused", queue)
+          end
+        }
+
+        queue_summaries = []
+        queues.each_with_index do |name, idx|
+          size = results[idx * 3]
+          last_item = results[idx * 3 + 1]
+          paused = results[idx * 3 + 2] > 0
+
+          latency = if last_item
+            job = Sidekiq.load_json(last_item)
+            calculate_latency(job)
+          else
+            0.0
+          end
+
+          queue_summaries << QueueSummary.new(name:, size:, latency:, paused:)
+        end
+
+        queue_summaries.sort_by { |qd| -qd.size }
+      end
+    end
+
     # O(1) redis calls
     # @api private
     def fetch_stats_fast!
@@ -100,19 +161,7 @@ module Sidekiq
           {}
         end
 
-        enqueued_at = job["enqueued_at"]
-        if enqueued_at
-          if enqueued_at.is_a?(Float)
-            # old format
-            now = Time.now.to_f
-            now - enqueued_at
-          else
-            now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
-            (now - enqueued_at) / 1000.0
-          end
-        else
-          0.0
-        end
+        calculate_latency(job)
       else
         0.0
       end
@@ -235,6 +284,7 @@ module Sidekiq
   #   end
   class Queue
     include Enumerable
+    include ApiUtils
 
     ##
     # Fetch all known queues within Redis.
@@ -245,6 +295,7 @@ module Sidekiq
     end
 
     attr_reader :name
+    alias_method :id, :name
 
     # @param name [String] the name of the queue
     def initialize(name = "default")
@@ -277,19 +328,7 @@ module Sidekiq
       return 0.0 unless entry
 
       job = Sidekiq.load_json(entry)
-      enqueued_at = job["enqueued_at"]
-      if enqueued_at
-        if enqueued_at.is_a?(Float)
-          # old format
-          now = Time.now.to_f
-          now - enqueued_at
-        else
-          now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
-          (now - enqueued_at) / 1000.0
-        end
-      else
-        0.0
-      end
+      calculate_latency(job)
     end
 
     def each
@@ -352,6 +391,8 @@ module Sidekiq
   # The job should be considered immutable but may be
   # removed from the queue via JobRecord#delete.
   class JobRecord
+    include ApiUtils
+
     # the parsed Hash of job data
     # @!attribute [r] Item
     attr_reader :item
@@ -478,17 +519,7 @@ module Sidekiq
     end
 
     def latency
-      timestamp = @item["enqueued_at"] || @item["created_at"]
-      if timestamp
-        if timestamp.is_a?(Float)
-          # old format
-          Time.now.to_f - timestamp
-        else
-          (::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) - timestamp) / 1000.0
-        end
-      else
-        0.0
-      end
+      calculate_latency(@item)
     end
 
     # Remove this job from the queue
@@ -553,17 +584,24 @@ module Sidekiq
   # could be the scheduled time for it to run (e.g. scheduled set),
   # or the expiration date after which the entry should be deleted (e.g. dead set).
   class SortedEntry < JobRecord
-    attr_reader :score
     attr_reader :parent
 
     # :nodoc:
     # @api private
     def initialize(parent, score, item)
       super(item)
-      @score = Float(score)
+      @score = score
       @parent = parent
     end
 
+    def score
+      Float(@score)
+    end
+
+    def id
+      "#{@score}|#{item["jid"]}"
+    end
+
     # The timestamp associated with this entry
     def at
       Time.at(score).utc
@@ -574,7 +612,7 @@ module Sidekiq
       if @value
         @parent.delete_by_value(@parent.name, @value)
       else
-        @parent.delete_by_jid(score, jid)
+        @parent.delete_by_jid(@score, jid)
       end
     end
 
@@ -583,7 +621,7 @@ module Sidekiq
     # @param at [Time] the new timestamp for this job
     def reschedule(at)
       Sidekiq.redis do |conn|
-        conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
+        conn.zincrby(@parent.name, at.to_f - score, Sidekiq.dump_json(@item))
       end
     end
 
@@ -1088,6 +1126,7 @@ module Sidekiq
     def identity
       self["identity"]
     end
+    alias_method :id, :identity
 
     # deprecated, use capsules below
     def queues
@@ -1161,6 +1200,10 @@ module Sidekiq
       self["quiet"] == "true"
     end
 
+    def leader?
+      Sidekiq.redis { |c| c.get("dear-leader") == identity }
+    end
+
     private
 
     def signal(sig)

data/lib/sidekiq/cli.rb

@@ -203,6 +203,7 @@ module Sidekiq # :nodoc:
       },
       # deprecated, use INFO
       "TTIN" => ->(cli) {
+        cli.logger.error { "DEPRECATED: Please use the INFO signal for backtraces, support for TTIN will be removed in Sidekiq 9.0." }
         Thread.list.each do |thread|
           cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
           if thread.backtrace

data/lib/sidekiq/config.rb

@@ -37,7 +37,7 @@ module Sidekiq
       reloader: proc { |&block| block.call },
       backtrace_cleaner: ->(backtrace) { backtrace },
       logged_job_attributes: ["bid", "tags"],
-      reap_connections: nil # TODO Enable by default
+      redis_idle_timeout: nil
     }
 
     ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
@@ -146,10 +146,9 @@ module Sidekiq
       @redis_config = @redis_config.merge(hash)
     end
 
-    def reap_idle_redis_connections(timeout = nil)
-      self[:reap_connections] = timeout
+    def reap_idle_redis_connections(timeout = 60)
+      self[:redis_idle_timeout] = timeout
     end
-    alias_method :reap, :reap_idle_redis_connections
 
     def redis_pool
       Thread.current[:sidekiq_redis_pool] || Thread.current[:sidekiq_capsule]&.redis_pool || local_redis_pool

data/lib/sidekiq/job.rb

@@ -226,6 +226,8 @@ module Sidekiq
         end
         return nil unless result
 
+        verify_json(item)
+
         # round-trip the payload via JSON
         msg = Sidekiq.load_json(Sidekiq.dump_json(item))
 

data/lib/sidekiq/job_retry.rb

@@ -178,10 +178,14 @@ module Sidekiq
         msg["error_backtrace"] = compress_backtrace(lines)
       end
 
-      return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
-
+      # retry_for and retry are mutually exclusive - if retry_for is set,
+      # we exclusively use duration-based retry logic and ignore count-based logic
       rf = msg["retry_for"]
-      return retries_exhausted(jobinst, msg, exception) if rf && (time_for(msg["failed_at"]) + rf) < Time.now
+      if rf
+        return retries_exhausted(jobinst, msg, exception) if (time_for(msg["failed_at"]) + rf) < Time.now
+      elsif count >= max_retry_attempts
+        return retries_exhausted(jobinst, msg, exception)
+      end
 
       strategy, delay = delay_for(jobinst, count, exception, msg)
       case strategy

data/lib/sidekiq/launcher.rb

@@ -142,10 +142,10 @@ module Sidekiq
       key = identity
       fails = procd = 0
 
-      idle = config[:reap_connections]
-      if idle
-        config.capsules.each_value { |cap| cap.local_redis_pool.reap(idle, &:close) }
-        config.local_redis_pool.reap(idle, &:close)
+      idle_timeout = config[:redis_idle_timeout]
+      if idle_timeout
+        config.capsules.each_value { |cap| cap.local_redis_pool.reap(idle_seconds: idle_timeout, &:close) }
+        config.local_redis_pool.reap(idle_seconds: idle_timeout, &:close)
       end
 
       begin

data/lib/sidekiq/scheduled.rb

@@ -72,6 +72,7 @@ module Sidekiq
       include Sidekiq::Component
 
       INITIAL_WAIT = 10
+      attr_accessor :rnd
 
       def initialize(config)
         @config = config
@@ -80,6 +81,7 @@ module Sidekiq
         @done = false
         @thread = nil
         @count_calls = 0
+        @rnd = Random.new
       end
 
       # Shut down this instance, will pause until the thread is dead.
@@ -151,11 +153,11 @@ module Sidekiq
 
         if count < 10
           # For small clusters, calculate a random interval that is ±50% the desired average.
-          interval * rand + interval.to_f / 2
+          interval * @rnd.rand + interval.to_f / 2
         else
           # With 10+ processes, we should have enough randomness to get decent polling
           # across the entire timespan
-          interval * rand * 2
+          interval * @rnd.rand * 2
         end
       end