sidekiq 8.0.10 → 8.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a831ed9ff5c1a11b447110fa77cbccf00fc44d3e3888fd5b1baa23cce5228ee9
-  data.tar.gz: 0b11c1b102a8ff8313992201b519b87e1e9446b84c72f23369e3390f671680f6
+  metadata.gz: 205ebd1ee2e6fdbe27b860b06a03147e96d77a3abc32481eb3bbf51ce01a64a6
+  data.tar.gz: 355195d5aadd03117e7eee7bdf144d193a4c9a41d4534b5f05ea6fad847cb7df
 SHA512:
-  metadata.gz: f6139796abb98a9bc67bf6498438f52e135b85ea6a360f86b45ddafd4f46d9d5567592940005d93bab5c3700430adfb5be340e08ca4a9e5cbde529d19a9c7dcc
-  data.tar.gz: '08ac62d0c45ca1a8216833ecfb07ad8f37ca3d1a1137198f0994af7cf0b107c842741a4f8b62ea8c3f59b42aa209418f65d9bcf6b57127b6ae6e09daf9509907'
+  metadata.gz: a592aa88c757173e70ee511388f15642e2aad1347df15252e972cea12e42ab91a0e8df702c3ff9e0069595bb68dae32a6d53318df390563a9ae7cc0df31852dd
+  data.tar.gz: d10b27d602e3c53fe79a85d21a59d0492831c32c8a67e2a344869c42e22e402ad591f82e3835e14c879a81fe8020430450fecd6a324d008bf33c4592dee23ea9

data/Changes.md

@@ -2,6 +2,64 @@
 
 [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
 
+8.1.3
+----------
+
+- Fix edge case leading to duplicate, concurrent execution [#6379]
+  If 2 Capsules process jobs from the same queue, long-running
+  jobs could run in parallel during process shutdown.
+- [SECURITY] Remove as much YAML usage as possible. [#6950]
+  Localization files in `web/locales` are now manually parsed.
+  Sidekiq::CLI will now only require YAML if you use a `-C` .yml file.
+
+8.1.2
+----------
+
+- Initial release for `kiq`, Sidekiq's official terminal UI:
+```
+bundle exec kiq
+```
+Use REDIS_URL or REDIS_PROVIDER to point `kiq` to Redis.
+- Mutation during iteration in `SortedSet#each` caused it to miss half of the jobs [#6936]
+- Fix edge case resulting in nil crash on /busy page [#6954]
+
+8.1.1
+----------
+
+- **DEPRECATION** `require 'sidekiq/testing'` and
+  `require 'sidekiq/testing/inline'`.
+  Add new `Sidekiq.testing!(mode)` API [#6931]
+  Requiring code should not enable process-wide changes.
+```ruby
+# Old, implicit
+require "sidekiq/testing"
+require "sidekiq/testing/inline"
+# New, more explicit
+Sidekiq.testing!(:fake)
+Sidekiq.testing!(:inline)
+```
+- Fix race condition with Stop button in UI [#6935]
+- Fix javascript error handler [#6893]
+
+8.1.0
+----------
+
+- `retry_for` and `retry` are now mutually exclusive [#6878, Saidbek]
+- `perform_inline` now enforces `strict_args!` [#6718, Saidbek]
+- Integrate Herb linting for ERB templates [#6760, Saidbek]
+- Remove CSRF code, use `Sec-Fetch-Site` header [#6874, deve1212]
+- Allow custom Web UI `assets_path` for CDN purposes [#6865, stanhu]
+- Upgrade to connection_pool 3.0
+- Allow idle connection reaping after N seconds.
+You can activate this **beta** feature like below.
+Feedback requested: is this feature stable and useful for you in production?
+This feature may or may not be enabled by default in Sidekiq 9.0.
+```ruby
+Sidekiq.configure_server do |cfg|
+  cfg.reap_idle_redis_connections(60)
+end
+```
+
 8.0.10
 ----------
 

data/README.md

@@ -90,13 +90,28 @@ Useful resources:
 * The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A.
 
 Every Thursday morning is Sidekiq Office Hour: I video chat and answer questions.
-See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
+See the [Sidekiq support page](https://sidekiq.org/support/) for details.
 
 Contributing
 -----------------
 
 See [the contributing guidelines](https://github.com/sidekiq/sidekiq/blob/main/.github/contributing.md).
 
+### ERB Linting with HERB
+
+This project uses [HERB](https://herb-tools.dev/) for ERB file linting and formatting. All ERB files have been renamed to use the `.html.erb` extension for better tooling support.
+
+**Local Development:**
+```bash
+# Run HERB linting
+bundle exec rake lint:herb
+# or
+bin/lint-herb
+```
+
+**CI Integration:**
+HERB linting is automatically run in CI to ensure all ERB files are properly formatted and free of parse errors.
+
 License
 -----------------
 

data/bin/kiq

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+# This requires the default gemset so Sidekiq Pro
+# and Sidekiq Enterprise can load any code extensions.
+Bundler.require(:default, :tui)
+
+require_relative "../lib/sidekiq/tui"
+
+# Run any load hooks registered during Bundler.require
+Sidekiq.loader.run_load_hooks(:tui)
+
+tt = Sidekiq::TUI.new(Sidekiq.default_configuration)
+
+RatatuiRuby.run do |tui|
+  tt.prepare(tui)
+  tt.run_loop
+end

data/bin/lint-herb

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# HERB Linting Script
+# Run this script to lint all ERB files in the project
+# Usage: bin/lint-herb
+
+require "bundler/setup"
+
+puts "🔍 Running HERB linting on ERB files..."
+puts
+
+exec("bundle exec herb analyze web/views -n --no-log-file")

data/lib/active_job/queue_adapters/sidekiq_adapter.rb

@@ -61,18 +61,21 @@ begin
 
         # @api private
         def enqueue(job)
-          job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(
-            wrapped: job.class,
-            queue: job.queue_name
-          ).perform_async(job.serialize)
+          # NB: Active Job only serializes keys it recognizes. We
+          # cannot set arbitrary key/values here.
+          options = {wrapped: job.class, queue: job.queue_name}
+          options[:profile] = job.profile if job.respond_to?(:profile) && !job.profile.nil?
+
+          wrapper = Sidekiq::ActiveJob::Wrapper.set(options)
+          job.provider_job_id = wrapper.perform_async(job.serialize)
         end
 
         # @api private
         def enqueue_at(job, timestamp)
-          job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(
-            wrapped: job.class,
-            queue: job.queue_name
-          ).perform_at(timestamp, job.serialize)
+          options = {wrapped: job.class, queue: job.queue_name}
+          options[:profile] = job.profile if job.respond_to?(:profile) && !job.profile.nil?
+
+          job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(options).perform_at(timestamp, job.serialize)
         end
 
         # @api private

data/lib/generators/sidekiq/job_generator.rb

@@ -14,7 +14,10 @@ module Sidekiq
       end
 
       def create_job_file
-        template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
+        template(
+          "job.rb.erb",
+          File.join(jobs_directory, class_path, "#{file_name}_job.rb")
+        )
       end
 
       def create_test_file
@@ -31,7 +34,8 @@ module Sidekiq
 
       def create_job_spec
         template_file = File.join(
-          "spec/sidekiq",
+          "spec",
+          jobs_directory.gsub("app/", ""),
           class_path,
           "#{file_name}_job_spec.rb"
         )
@@ -40,7 +44,8 @@ module Sidekiq
 
       def create_job_test
         template_file = File.join(
-          "test/sidekiq",
+          "test",
+          jobs_directory.gsub("app/", ""),
           class_path,
           "#{file_name}_job_test.rb"
         )
@@ -54,6 +59,13 @@ module Sidekiq
       def test_framework
         ::Rails.application.config.generators.options[:rails][:test_framework]
       end
+
+      # Can be set in an initializer or in application configuration
+      # with Rails.application.config.generators.options[:sidekiq][:jobs_directory] = "app/jobs"
+      # to change the directory that the job files are generated in to.
+      def jobs_directory
+        ::Rails.application.config.generators.options[:sidekiq].fetch(:jobs_directory, "app/sidekiq")
+      end
     end
   end
 end

data/lib/sidekiq/api.rb

@@ -17,12 +17,37 @@ require "sidekiq/metrics/query"
 #
 
 module Sidekiq
+  module ApiUtils
+    # @api private
+    # Calculate the latency in seconds for a job based on its enqueued timestamp
+    # @param job [Hash] the job hash
+    # @return [Float] latency in seconds
+    def calculate_latency(job)
+      timestamp = job["enqueued_at"] || job["created_at"]
+      return 0.0 unless timestamp
+
+      if timestamp.is_a?(Float)
+        # old format
+        Time.now.to_f - timestamp
+      else
+        now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
+        (now - timestamp) / 1000.0
+      end
+    end
+  end
+
   # Retrieve runtime statistics from Redis regarding
   # this Sidekiq cluster.
   #
   #   stat = Sidekiq::Stats.new
   #   stat.processed
   class Stats
+    QueueSummary = Data.define(:name, :size, :latency, :paused) do
+      alias_method :paused?, :paused
+    end
+
+    include ApiUtils
+
     def initialize
       fetch_stats_fast!
     end
@@ -63,6 +88,7 @@ module Sidekiq
       stat :default_queue_latency
     end
 
+    # @return [Hash{String => Integer}] a hash of queue names to their lengths
     def queues
       Sidekiq.redis do |conn|
         queues = conn.sscan("queues").to_a
@@ -78,6 +104,41 @@ module Sidekiq
       end
     end
 
+    # More detailed information about each queue: name, size, latency, paused status
+    # @return [Array<QueueSummary>]
+    def queue_summaries
+      Sidekiq.redis do |conn|
+        queues = conn.sscan("queues").to_a
+        return [] if queues.empty?
+
+        results = conn.pipelined { |pipeline|
+          queues.each do |queue|
+            pipeline.llen("queue:#{queue}")
+            pipeline.lindex("queue:#{queue}", -1)
+            pipeline.sismember("paused", queue)
+          end
+        }
+
+        queue_summaries = []
+        queues.each_with_index do |name, idx|
+          size = results[idx * 3]
+          last_item = results[idx * 3 + 1]
+          paused = results[idx * 3 + 2] > 0
+
+          latency = if last_item
+            job = Sidekiq.load_json(last_item)
+            calculate_latency(job)
+          else
+            0.0
+          end
+
+          queue_summaries << QueueSummary.new(name:, size:, latency:, paused:)
+        end
+
+        queue_summaries.sort_by { |qd| -qd.size }
+      end
+    end
+
     # O(1) redis calls
     # @api private
     def fetch_stats_fast!
@@ -100,19 +161,7 @@ module Sidekiq
           {}
         end
 
-        enqueued_at = job["enqueued_at"]
-        if enqueued_at
-          if enqueued_at.is_a?(Float)
-            # old format
-            now = Time.now.to_f
-            now - enqueued_at
-          else
-            now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
-            (now - enqueued_at) / 1000.0
-          end
-        else
-          0.0
-        end
+        calculate_latency(job)
       else
         0.0
       end
@@ -235,6 +284,7 @@ module Sidekiq
   #   end
   class Queue
     include Enumerable
+    include ApiUtils
 
     ##
     # Fetch all known queues within Redis.
@@ -245,6 +295,7 @@ module Sidekiq
     end
 
     attr_reader :name
+    alias_method :id, :name
 
     # @param name [String] the name of the queue
     def initialize(name = "default")
@@ -277,19 +328,7 @@ module Sidekiq
       return 0.0 unless entry
 
       job = Sidekiq.load_json(entry)
-      enqueued_at = job["enqueued_at"]
-      if enqueued_at
-        if enqueued_at.is_a?(Float)
-          # old format
-          now = Time.now.to_f
-          now - enqueued_at
-        else
-          now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
-          (now - enqueued_at) / 1000.0
-        end
-      else
-        0.0
-      end
+      calculate_latency(job)
     end
 
     def each
@@ -352,6 +391,8 @@ module Sidekiq
   # The job should be considered immutable but may be
   # removed from the queue via JobRecord#delete.
   class JobRecord
+    include ApiUtils
+
     # the parsed Hash of job data
     # @!attribute [r] Item
     attr_reader :item
@@ -478,17 +519,7 @@ module Sidekiq
     end
 
     def latency
-      timestamp = @item["enqueued_at"] || @item["created_at"]
-      if timestamp
-        if timestamp.is_a?(Float)
-          # old format
-          Time.now.to_f - timestamp
-        else
-          (::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) - timestamp) / 1000.0
-        end
-      else
-        0.0
-      end
+      calculate_latency(@item)
     end
 
     # Remove this job from the queue
@@ -553,17 +584,24 @@ module Sidekiq
   # could be the scheduled time for it to run (e.g. scheduled set),
   # or the expiration date after which the entry should be deleted (e.g. dead set).
   class SortedEntry < JobRecord
-    attr_reader :score
     attr_reader :parent
 
     # :nodoc:
     # @api private
     def initialize(parent, score, item)
       super(item)
-      @score = Float(score)
+      @score = score
       @parent = parent
     end
 
+    def score
+      Float(@score)
+    end
+
+    def id
+      "#{@score}|#{item["jid"]}"
+    end
+
     # The timestamp associated with this entry
     def at
       Time.at(score).utc
@@ -574,7 +612,7 @@ module Sidekiq
       if @value
         @parent.delete_by_value(@parent.name, @value)
       else
-        @parent.delete_by_jid(score, jid)
+        @parent.delete_by_jid(@score, jid)
       end
     end
 
@@ -583,7 +621,7 @@ module Sidekiq
     # @param at [Time] the new timestamp for this job
     def reschedule(at)
       Sidekiq.redis do |conn|
-        conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
+        conn.zincrby(@parent.name, at.to_f - score, Sidekiq.dump_json(@item))
       end
     end
 
@@ -619,38 +657,8 @@ module Sidekiq
 
     private
 
-    def remove_job
-      Sidekiq.redis do |conn|
-        results = conn.multi { |transaction|
-          transaction.zrange(parent.name, score, score, "BYSCORE")
-          transaction.zremrangebyscore(parent.name, score, score)
-        }.first
-
-        if results.size == 1
-          yield results.first
-        else
-          # multiple jobs with the same score
-          # find the one with the right JID and push it
-          matched, nonmatched = results.partition { |message|
-            if message.index(jid)
-              msg = Sidekiq.load_json(message)
-              msg["jid"] == jid
-            else
-              false
-            end
-          }
-
-          msg = matched.first
-          yield msg if msg
-
-          # push the rest back onto the sorted set
-          conn.multi do |transaction|
-            nonmatched.each do |message|
-              transaction.zadd(parent.name, score.to_f.to_s, message)
-            end
-          end
-        end
-      end
+    def remove_job(&)
+      parent.remove_job(self, &)
     end
   end
 
@@ -819,6 +827,46 @@ module Sidekiq
       nil
     end
 
+    def remove_job(entry)
+      score = entry.score
+      jid = entry.jid
+      Sidekiq.redis do |conn|
+        results = conn.multi { |transaction|
+          transaction.zrange(name, score, score, "BYSCORE")
+          transaction.zremrangebyscore(name, score, score)
+        }.first
+
+        if results.size == 1
+          yield results.first
+          @_size -= 1
+        else
+          # multiple jobs with the same score
+          # find the one with the right JID and push it
+          matched, nonmatched = results.partition { |message|
+            if message.index(jid)
+              msg = Sidekiq.load_json(message)
+              msg["jid"] == jid
+            else
+              false
+            end
+          }
+
+          msg = matched.first
+          if msg
+            yield msg
+            @_size -= 1
+          end
+
+          # push the rest back onto the sorted set
+          conn.multi do |transaction|
+            nonmatched.each do |message|
+              transaction.zadd(name, score.to_f.to_s, message)
+            end
+          end
+        end
+      end
+    end
+
     # :nodoc:
     # @api private
     def delete_by_value(name, value)
@@ -992,19 +1040,20 @@ module Sidekiq
         # you'll be happier this way
         conn.pipelined do |pipeline|
           procs.each do |key|
-            pipeline.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
+            pipeline.hmget(key, "info", "concurrency", "busy", "beat", "quiet", "rss", "rtt_us")
           end
         end
       }
 
-      result.each do |info, busy, beat, quiet, rss, rtt_us|
+      result.each do |info, concurrency, busy, beat, quiet, rss, rtt_us|
         # If a process is stopped between when we query Redis for `procs` and
         # when we query for `result`, we will have an item in `result` that is
         # composed of `nil` values.
         next if info.nil?
 
         hash = Sidekiq.load_json(info)
-        yield Process.new(hash.merge("busy" => busy.to_i,
+        yield Process.new(hash.merge("concurrency" => concurrency.to_i,
+          "busy" => busy.to_i,
           "beat" => beat.to_f,
           "quiet" => quiet,
           "rss" => rss.to_i,
@@ -1088,6 +1137,7 @@ module Sidekiq
     def identity
       self["identity"]
     end
+    alias_method :id, :identity
 
     # deprecated, use capsules below
     def queues
@@ -1161,6 +1211,10 @@ module Sidekiq
       self["quiet"] == "true"
     end
 
+    def leader?
+      Sidekiq.redis { |c| c.get("dear-leader") == identity }
+    end
+
     private
 
     def signal(sig)

data/lib/sidekiq/capsule.rb

@@ -51,7 +51,6 @@ module Sidekiq
     end
 
     def stop
-      fetcher&.bulk_requeue([])
     end
 
     # Sidekiq checks queues in three modes:

data/lib/sidekiq/cli.rb

@@ -2,7 +2,6 @@
 
 $stdout.sync = true
 
-require "yaml"
 require "optparse"
 require "erb"
 require "fileutils"
@@ -203,6 +202,7 @@ module Sidekiq # :nodoc:
       },
       # deprecated, use INFO
       "TTIN" => ->(cli) {
+        cli.logger.error { "DEPRECATED: Please use the INFO signal for backtraces, support for TTIN will be removed in Sidekiq 9.0." }
         Thread.list.each do |thread|
           cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
           if thread.backtrace
@@ -408,6 +408,7 @@ module Sidekiq # :nodoc:
     def parse_config(path)
       erb = ERB.new(File.read(path), trim_mode: "-")
       erb.filename = File.expand_path(path)
+      require "yaml"
       opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
 
       if opts.respond_to? :deep_symbolize_keys!

data/lib/sidekiq/client.rb

@@ -132,12 +132,14 @@ module Sidekiq
     #   push_bulk('class' => MyJob, 'args' => (1..100_000).to_a, batch_size: 1_000)
     #
     def push_bulk(items)
-      batch_size = items.delete(:batch_size) || items.delete("batch_size") || 1_000
       args = items["args"]
       at = items.delete("at") || items.delete(:at)
       raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
       raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
+      # Use a smaller batch size by default for scheduled jobs since adding to sorted sets is more costly.
+      batch_size = items.delete(:batch_size) || items.delete("batch_size") || (at ? 100 : 1_000)
+
       jid = items.delete("jid")
       raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
 

data/lib/sidekiq/component.rb

@@ -57,6 +57,9 @@ module Sidekiq
     end
 
     def tid
+      # We XOR with PID to ensure Thread IDs changes after fork.
+      # I'm unclear why we don't multiply the two values to better guarantee
+      # a unique value but it's been this way for quite a while now. #3685
       Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
     end
 

data/lib/sidekiq/config.rb

@@ -37,7 +37,7 @@ module Sidekiq
       reloader: proc { |&block| block.call },
       backtrace_cleaner: ->(backtrace) { backtrace },
       logged_job_attributes: ["bid", "tags"],
-      reap_connections: nil # TODO Enable by default
+      redis_idle_timeout: nil
     }
 
     ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
@@ -146,10 +146,9 @@ module Sidekiq
       @redis_config = @redis_config.merge(hash)
     end
 
-    def reap_idle_redis_connections(timeout = nil)
-      self[:reap_connections] = timeout
+    def reap_idle_redis_connections(timeout = 60)
+      self[:redis_idle_timeout] = timeout
     end
-    alias_method :reap, :reap_idle_redis_connections
 
     def redis_pool
       Thread.current[:sidekiq_redis_pool] || Thread.current[:sidekiq_capsule]&.redis_pool || local_redis_pool
@@ -256,7 +255,7 @@ module Sidekiq
     # Register a proc to handle any error which occurs within the Sidekiq process.
     #
     #   Sidekiq.configure_server do |config|
-    #     config.error_handlers << proc {|ex,ctx_hash| MyErrorService.notify(ex, ctx_hash) }
+    #     config.error_handlers << proc {|ex,ctx_hash,config| MyErrorService.notify(ex, ctx_hash) }
     #   end
     #
     # The default error handler logs errors to @logger.

data/lib/sidekiq/job.rb

@@ -226,6 +226,8 @@ module Sidekiq
         end
         return nil unless result
 
+        verify_json(item)
+
         # round-trip the payload via JSON
         msg = Sidekiq.load_json(Sidekiq.dump_json(item))
 

data/lib/sidekiq/job_retry.rb

@@ -178,10 +178,14 @@ module Sidekiq
         msg["error_backtrace"] = compress_backtrace(lines)
       end
 
-      return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
-
+      # retry_for and retry are mutually exclusive - if retry_for is set,
+      # we exclusively use duration-based retry logic and ignore count-based logic
       rf = msg["retry_for"]
-      return retries_exhausted(jobinst, msg, exception) if rf && (time_for(msg["failed_at"]) + rf) < Time.now
+      if rf
+        return retries_exhausted(jobinst, msg, exception) if (time_for(msg["failed_at"]) + rf) < Time.now
+      elsif count >= max_retry_attempts
+        return retries_exhausted(jobinst, msg, exception)
+      end
 
       strategy, delay = delay_for(jobinst, count, exception, msg)
       case strategy