sidekiq 8.0.8 → 8.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a1e8c90888b4135f21e4dfef66fd9ad9ac8c72ab267599a8452f4bd172234a2
-  data.tar.gz: fa83e70c818fc3b25441e946c1a093a2d9a5f4515ee2bd88522cbafcb8097973
+  metadata.gz: 63658318932ac6b7045211590b07f84ed9e1e6398f908ace790a0df20d5ac1f0
+  data.tar.gz: 0c3f5e69ada7529bdac7392acb1c56ef2a19b8ed13210be8c36aac19858c6df4
 SHA512:
-  metadata.gz: 0ea8ac9585538723d941af6ce9150933669705cf1e3459dcea8989d073d234c4e233e74314a5a682e3111e1290a69671051e847b5ee97970c1edd08293e14bbc
-  data.tar.gz: 0b253cd035132f786613fbb97440f5d16f48717aedaa7f50865e0e261ce0fc17b64885bea3cdeb5aabe72258c06ed013891d83655bdcf842fc1a261d1b0aa985
+  metadata.gz: 9a5e95a2faccbbf42f8e651c4882c9495ad72b0500bb8fe77fa2a93fb4aa6f8cb96fc820621cd1910fb4b00accf35d75a99d548b3cc8ccdd41a18b5620a9ad73
+  data.tar.gz: 1bc66490ee07c548c71f680fb280e876efa7eeb8c6f234277df73211ce05c5ae29f5c9bc13064972da945568d1df6634252a6c22a7c5298bdd436b7266246c53

data/Changes.md

@@ -2,6 +2,48 @@
 
 [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
 
+8.1.0
+----------
+
+- `retry_for` and `retry` are now mutually exclusive [#6878, Saidbek]
+- `perform_inline` now enforces `strict_args!` [#6718, Saidbek]
+- Integrate Herb linting for ERB templates [#6760, Saidbek]
+- Remove CSRF code, use `Sec-Fetch-Site` header [#6874, deve1212]
+- Allow custom Web UI `assets_path` for CDN purposes [#6865, stanhu]
+- Upgrade to connection_pool 3.0
+- Allow idle connection reaping after N seconds.
+You can activate this **beta** feature like below.
+Feedback requested: is this feature stable and useful for you in production?
+This feature may or may not be enabled by default in Sidekiq 9.0.
+```ruby
+Sidekiq.configure_server do |cfg|
+  cfg.reap_idle_redis_connections(60)
+end
+```
+
+8.0.10
+----------
+
+- Add confirm dialog for Delete All buttons in Web UI [#6853]
+- Adjust scheduler to run closer to poll average [#6866]
+- Forward compatibility changes for connection_pool 3.0.0
+- Backwards compatibility fix for <8.0.9 process data in Redis [#6870]
+- Backtrace dump can now be triggered with the INFO signal, since Puma uses the
+  same signal [#6857]
+
+8.0.9
+----------
+
+- Implement idle Redis connection reaping, will be activated in 8.1 [#6663]
+- Updated `Sidekiq::Process` API to provide capsule data. The `queues` and `weights` 
+  data will be removed from Redis in Sidekiq 8.1, as this data can now be found in the
+  `capsules` element. [#6295]
+- Restore bulk action buttons on Scheduled, Retry and Dead tabs [#6833, deve1212]
+- Support logging additional job attributes [#6846, bschrag620]
+- Fix display of long job args [#6836]
+- Create development lifecycle (`docs/sdlc.md`) and security (`docs/SECURITY.md`) policy
+  documentation for Sidekiq's current workflows
+
 8.0.8
 ----------
 

data/README.md

@@ -97,6 +97,21 @@ Contributing
 
 See [the contributing guidelines](https://github.com/sidekiq/sidekiq/blob/main/.github/contributing.md).
 
+### ERB Linting with HERB
+
+This project uses [HERB](https://herb-tools.dev/) for ERB file linting and formatting. All ERB files have been renamed to use the `.html.erb` extension for better tooling support.
+
+**Local Development:**
+```bash
+# Run HERB linting
+bundle exec rake lint:herb
+# or
+bin/lint-herb
+```
+
+**CI Integration:**
+HERB linting is automatically run in CI to ensure all ERB files are properly formatted and free of parse errors.
+
 License
 -----------------
 

data/bin/lint-herb

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# HERB Linting Script
+# Run this script to lint all ERB files in the project
+# Usage: bin/lint-herb
+
+require "bundler/setup"
+
+puts "🔍 Running HERB linting on ERB files..."
+puts
+
+exec("bundle exec herb analyze web/views -n --no-log-file")

data/lib/generators/sidekiq/templates/job.rb.erb

@@ -6,4 +6,4 @@ class <%= class_name %>Job
     # Do something
   end
 end
-<% end -%>
+<% end -%>

data/lib/sidekiq/api.rb

@@ -1059,9 +1059,9 @@ module Sidekiq
   #   'started_at' => <process start time>,
   #   'pid' => 12345,
   #   'tag' => 'myapp'
-  #   'concurrency' => 25,
-  #   'queues' => ['default', 'low'],
-  #   'busy' => 10,
+  #   'concurrency' => 5,
+  #   'capsules' => {"default" => {"mode" => "weighted", "concurrency" => 5, "weights" => {"default" => 2, "low" => 1}}},
+  #   'busy' => 3,
   #   'beat' => <last heartbeat>,
   #   'identity' => <unique string identifying the process>,
   #   'embedded' => true,
@@ -1089,12 +1089,35 @@ module Sidekiq
       self["identity"]
     end
 
+    # deprecated, use capsules below
     def queues
-      self["queues"]
+      # Backwards compatibility with <8.0.8
+      if !self["capsules"]
+        self["queues"]
+      else
+        capsules.values.flat_map { |x| x["weights"].keys }.uniq
+      end
     end
 
+    # deprecated, use capsules below
     def weights
-      self["weights"]
+      # Backwards compatibility with <8.0.8
+      if !self["capsules"]
+        self["weights"]
+      else
+        hash = {}
+        capsules.values.each do |cap|
+          # Note: will lose data if two capsules are processing the same named queue
+          cap["weights"].each_pair do |queue, weight|
+            hash[queue] = weight
+          end
+        end
+        hash
+      end
+    end
+
+    def capsules
+      self["capsules"]
     end
 
     def version

data/lib/sidekiq/capsule.rb

@@ -38,6 +38,10 @@ module Sidekiq
       @mode = :strict
     end
 
+    def to_h
+      {concurrency: concurrency, mode: mode, weights: weights}
+    end
+
     def fetcher
       @fetcher ||= begin
         instance = (config[:fetch_class] || Sidekiq::BasicFetch).new(self)

data/lib/sidekiq/cli.rb

@@ -49,7 +49,7 @@ module Sidekiq # :nodoc:
       logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
 
       self_read, self_write = IO.pipe
-      sigs = %w[INT TERM TTIN TSTP]
+      sigs = %w[INT TERM INFO TTIN TSTP]
       # USR1 and USR2 don't work on the JVM
       sigs << "USR2" if Sidekiq.pro? && !jruby?
       sigs.each do |sig|
@@ -201,7 +201,19 @@ module Sidekiq # :nodoc:
         cli.logger.info "Received TSTP, no longer accepting new work"
         cli.launcher.quiet
       },
+      # deprecated, use INFO
       "TTIN" => ->(cli) {
+        cli.logger.error { "DEPRECATED: Please use the INFO signal for backtraces, support for TTIN will be removed in Sidekiq 9.0." }
+        Thread.list.each do |thread|
+          cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
+          if thread.backtrace
+            cli.logger.warn thread.backtrace.join("\n")
+          else
+            cli.logger.warn "<no backtrace available>"
+          end
+        end
+      },
+      "INFO" => ->(cli) {
         Thread.list.each do |thread|
           cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
           if thread.backtrace
@@ -212,12 +224,12 @@ module Sidekiq # :nodoc:
         end
       }
     }
-    UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
-    SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
 
     def handle_signal(sig)
       logger.debug "Got #{sig} signal"
-      SIGNAL_HANDLERS[sig].call(self)
+      hndlr = SIGNAL_HANDLERS[sig]
+      hndlr ? hndlr.call(self) :
+        logger.warn("No #{sig} signal handler registered, ignoring")
     end
 
     private

data/lib/sidekiq/config.rb

@@ -35,7 +35,9 @@ module Sidekiq
       dead_max_jobs: 10_000,
       dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
       reloader: proc { |&block| block.call },
-      backtrace_cleaner: ->(backtrace) { backtrace }
+      backtrace_cleaner: ->(backtrace) { backtrace },
+      logged_job_attributes: ["bid", "tags"],
+      redis_idle_timeout: nil
     }
 
     ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
@@ -144,11 +146,15 @@ module Sidekiq
       @redis_config = @redis_config.merge(hash)
     end
 
+    def reap_idle_redis_connections(timeout = 60)
+      self[:redis_idle_timeout] = timeout
+    end
+
     def redis_pool
       Thread.current[:sidekiq_redis_pool] || Thread.current[:sidekiq_capsule]&.redis_pool || local_redis_pool
     end
 
-    private def local_redis_pool
+    def local_redis_pool
       # this is our internal client/housekeeping pool. each capsule has its
       # own pool for executing threads.
       @redis ||= new_redis_pool(10, "internal")

data/lib/sidekiq/fetch.rb

@@ -7,6 +7,7 @@ require "sidekiq/capsule"
 module Sidekiq # :nodoc:
   class BasicFetch
     include Sidekiq::Component
+
     # We want the fetch operation to timeout every few seconds so the thread
     # can check if the process is shutting down.
     TIMEOUT = 2

data/lib/sidekiq/job/iterable.rb

@@ -55,7 +55,7 @@ module Sidekiq
       def cancel!
         return @_cancelled if cancelled?
 
-        key = "it-#{jid}"
+        key = iteration_key
         _, result, _ = Sidekiq.redis do |c|
           c.pipelined do |p|
             p.hsetnx(key, "cancelled", Time.now.to_i)
@@ -177,7 +177,7 @@ module Sidekiq
       private
 
       def is_cancelled?
-        @_cancelled = Sidekiq.redis { |c| c.hget("it-#{jid}", "cancelled") }
+        @_cancelled = Sidekiq.redis { |c| c.hget(iteration_key, "cancelled") }
       end
 
       def fetch_previous_iteration_state

data/lib/sidekiq/job.rb

@@ -226,6 +226,8 @@ module Sidekiq
         end
         return nil unless result
 
+        verify_json(item)
+
         # round-trip the payload via JSON
         msg = Sidekiq.load_json(Sidekiq.dump_json(item))
 

data/lib/sidekiq/job_logger.rb

@@ -29,8 +29,10 @@ module Sidekiq
         jid: job_hash["jid"],
         class: job_hash["wrapped"] || job_hash["class"]
       }
-      h[:bid] = job_hash["bid"] if job_hash.has_key?("bid")
-      h[:tags] = job_hash["tags"] if job_hash.has_key?("tags")
+
+      @config[:logged_job_attributes].each do |attr|
+        h[attr.to_sym] = job_hash[attr] if job_hash.has_key?(attr)
+      end
 
       Thread.current[:sidekiq_context] = h
       level = job_hash["log_level"]

data/lib/sidekiq/job_retry.rb

@@ -178,10 +178,14 @@ module Sidekiq
         msg["error_backtrace"] = compress_backtrace(lines)
       end
 
-      return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
-
+      # retry_for and retry are mutually exclusive - if retry_for is set,
+      # we exclusively use duration-based retry logic and ignore count-based logic
       rf = msg["retry_for"]
-      return retries_exhausted(jobinst, msg, exception) if rf && (time_for(msg["failed_at"]) + rf) < Time.now
+      if rf
+        return retries_exhausted(jobinst, msg, exception) if (time_for(msg["failed_at"]) + rf) < Time.now
+      elsif count >= max_retry_attempts
+        return retries_exhausted(jobinst, msg, exception)
+      end
 
       strategy, delay = delay_for(jobinst, count, exception, msg)
       case strategy

data/lib/sidekiq/launcher.rb

@@ -142,6 +142,12 @@ module Sidekiq
       key = identity
       fails = procd = 0
 
+      idle_timeout = config[:redis_idle_timeout]
+      if idle_timeout
+        config.capsules.each_value { |cap| cap.local_redis_pool.reap(idle_seconds: idle_timeout, &:close) }
+        config.local_redis_pool.reap(idle_seconds: idle_timeout, &:close)
+      end
+
       begin
         flush_stats
 
@@ -214,11 +220,11 @@ module Sidekiq
       # Log a warning if it's a disaster.
       if RTT_READINGS.all? { |x| x > RTT_WARNING_LEVEL }
         logger.warn <<~EOM
-          Your Redis network connection is performing extremely poorly.
+          Your Redis network connection appears to be performing poorly.
           Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
-          Ensure Redis is running in the same AZ or datacenter as Sidekiq.
-          If these values are close to 100,000, that means your Sidekiq process may be
-          CPU-saturated; reduce your concurrency and/or see https://github.com/sidekiq/sidekiq/discussions/5039
+          Ensure Redis is running in the same AZ or datacenter as Sidekiq and that
+          your Sidekiq process is not CPU-saturated; reduce your concurrency and/or
+          see https://github.com/sidekiq/sidekiq/discussions/5039
         EOM
         RTT_READINGS.reset
       end
@@ -252,8 +258,15 @@ module Sidekiq
         "pid" => ::Process.pid,
         "tag" => @config[:tag] || "",
         "concurrency" => @config.total_concurrency,
+        "capsules" => @config.capsules.each_with_object({}) { |(name, cap), memo|
+          memo[name] = cap.to_h
+        },
+        #####
+        # TODO deprecated, remove in 9.0
+        # This data is now found in the `capsules` element above
         "queues" => @config.capsules.values.flat_map { |cap| cap.queues }.uniq,
-        "weights" => to_weights,
+        "weights" => @config.capsules.values.map(&:weights),
+        #####
         "labels" => @config[:labels].to_a,
         "identity" => identity,
         "version" => Sidekiq::VERSION,
@@ -261,10 +274,6 @@ module Sidekiq
       }
     end
 
-    def to_weights
-      @config.capsules.values.map(&:weights)
-    end
-
     def to_json
       # this data changes infrequently so dump it to a string
       # now so we don't need to dump it every heartbeat.

data/lib/sidekiq/middleware/i18n.rb

@@ -11,6 +11,7 @@ module Sidekiq::Middleware::I18n
   # to be sent to Sidekiq.
   class Client
     include Sidekiq::ClientMiddleware
+
     def call(_jobclass, job, _queue, _redis)
       job["locale"] ||= I18n.locale
       yield
@@ -20,6 +21,7 @@ module Sidekiq::Middleware::I18n
   # Pull the msg locale out and set the current thread to use it.
   class Server
     include Sidekiq::ServerMiddleware
+
     def call(_jobclass, job, _queue, &block)
       I18n.with_locale(job.fetch("locale", I18n.default_locale), &block)
     end

data/lib/sidekiq/monitor.rb

@@ -49,14 +49,10 @@ class Sidekiq::Monitor
       puts "---- Processes (#{process_set.size}) ----"
       process_set.each_with_index do |process, index|
         # Keep compatibility with legacy versions since we don't want to break sidekiqmon during rolling upgrades or downgrades.
-        #
-        # Before:
-        #   ["default", "critical"]
-        #
-        # After:
-        #   {"default" => 1, "critical" => 10}
         queues =
-          if process["weights"]
+          if process["capsules"] # 8.0.6+
+            process["capsules"].values.map { |x| x["weights"].keys.join(", ") }
+          elsif process["weights"]
             process["weights"].sort_by { |queue| queue[0] }.map { |capsule| capsule.map { |name, weight| (weight > 0) ? "#{name}: #{weight}" : name }.join(", ") }
           else
             process["queues"].sort
@@ -105,7 +101,7 @@ class Sidekiq::Monitor
           out << line
           line = " " * pad
         end
-        line << value + ", "
+        line << value + "; "
       end
       out << line[0..-3]
       out.join("\n")

data/lib/sidekiq/profiler.rb

@@ -11,6 +11,7 @@ module Sidekiq
     }
 
     include Sidekiq::Component
+
     def initialize(config)
       @config = config
       @vernier_output_dir = ENV.fetch("VERNIER_OUTPUT_DIR") { Dir.tmpdir }

data/lib/sidekiq/redis_connection.rb

@@ -23,7 +23,7 @@ module Sidekiq
 
         size = symbolized_options.delete(:size) || 5
         pool_timeout = symbolized_options.delete(:pool_timeout) || 1
-        pool_name = symbolized_options.delete(:pool_name)
+        symbolized_options.delete(:pool_name)
 
         # Default timeout in redis-client is 1 second, which can be too aggressive
         # if the Sidekiq process is CPU-bound. With 10-15 threads and a thread quantum of 100ms,
@@ -33,7 +33,7 @@ module Sidekiq
         symbolized_options[:timeout] ||= 3
 
         redis_config = Sidekiq::RedisClientAdapter.new(symbolized_options)
-        ConnectionPool.new(timeout: pool_timeout, size: size, name: pool_name) do
+        ConnectionPool.new(timeout: pool_timeout, size: size) do
           redis_config.new_client
         end
       end

data/lib/sidekiq/ring_buffer.rb

@@ -6,6 +6,7 @@ module Sidekiq
   class RingBuffer
     include Enumerable
     extend Forwardable
+
     def_delegators :@buf, :[], :each, :size
 
     def initialize(size, default = 0)

data/lib/sidekiq/scheduled.rb

@@ -72,6 +72,7 @@ module Sidekiq
       include Sidekiq::Component
 
       INITIAL_WAIT = 10
+      attr_accessor :rnd
 
       def initialize(config)
         @config = config
@@ -80,6 +81,7 @@ module Sidekiq
         @done = false
         @thread = nil
         @count_calls = 0
+        @rnd = Random.new
       end
 
       # Shut down this instance, will pause until the thread is dead.
@@ -115,9 +117,9 @@ module Sidekiq
       private
 
       def wait
-        @sleeper.pop(random_poll_interval)
+        @sleeper.pop(timeout: random_poll_interval)
       rescue Timeout::Error
-        # expected
+        # TODO move to exception: false
       rescue => ex
         # if poll_interval_average hasn't been calculated yet, we can
         # raise an error trying to reach Redis.
@@ -151,11 +153,11 @@ module Sidekiq
 
         if count < 10
           # For small clusters, calculate a random interval that is ±50% the desired average.
-          interval * rand + interval.to_f / 2
+          interval * @rnd.rand + interval.to_f / 2
         else
           # With 10+ processes, we should have enough randomness to get decent polling
           # across the entire timespan
-          interval * rand
+          interval * @rnd.rand * 2
         end
       end
 
@@ -223,7 +225,7 @@ module Sidekiq
         total += INITIAL_WAIT unless @config[:poll_interval_average]
         total += (5 * rand)
 
-        @sleeper.pop(total)
+        @sleeper.pop(timeout: total)
       rescue Timeout::Error
       ensure
         # periodically clean out the `processes` set in Redis which can collect

data/lib/sidekiq/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "8.0.8"
+  VERSION = "8.1.0"
   MAJOR = 8
 
   def self.gem_version

data/lib/sidekiq/web/action.rb

@@ -117,7 +117,7 @@ module Sidekiq
         if content.is_a? Symbol
           unless respond_to?(:"_erb_#{content}")
             views = options[:views] || Web.views
-            filename = "#{views}/#{content}.erb"
+            filename = "#{views}/#{content}.html.erb"
             src = ERB.new(File.read(filename)).src
 
             # Need to use lineno less by 1 because erb generates a

data/lib/sidekiq/web/application.rb

@@ -318,6 +318,16 @@ module Sidekiq
         redirect_with_query("#{root_path}scheduled")
       end
 
+      post "/scheduled/all/delete" do
+        Sidekiq::ScheduledSet.new.clear
+        redirect "#{root_path}scheduled"
+      end
+
+      post "/scheduled/all/add_to_queue" do
+        Sidekiq::ScheduledSet.new.each(&:add_to_queue)
+        redirect "#{root_path}scheduled"
+      end
+
       get "/dashboard/stats" do
         redirect "#{root_path}stats"
       end

data/lib/sidekiq/web/config.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "sidekiq/web/csrf_protection"
-
 module Sidekiq
   class Web
     ##
@@ -24,10 +22,7 @@ module Sidekiq
         # and very difficult for us to vendor or provide ourselves. If you are worried
         # about data security and wish to self-host, you can change these URLs.
         profile_view_url: "https://profiler.firefox.com/public/%s",
-        profile_store_url: "https://api.profiler.firefox.com/compressed-store",
-        # Will be false in Sidekiq 9.0.
-        # CSRF is unnecessary if you are using SameSite=(Strict|Lax) cookies.
-        csrf: true
+        profile_store_url: "https://api.profiler.firefox.com/compressed-store"
       }
 
       ##
@@ -54,11 +49,13 @@ module Sidekiq
 
       # Adds the "Back to App" link in the header
       attr_accessor :app_url
+      attr_accessor :assets_path
 
       def initialize
         @options = OPTIONS.dup
         @locales = LOCALES
         @views = VIEWS
+        @assets_path = ASSETS
         @tabs = DEFAULT_TABS.dup
         @middlewares = []
         @custom_job_info_rows = []

data/lib/sidekiq/web/helpers.rb

@@ -122,8 +122,8 @@ module Sidekiq
       resultset
     end
 
-    def filtering(which)
-      erb(:filtering, locals: {which: which})
+    def filtering(which, placeholder_key: "AnyJobContent", label_key: "Filter")
+      erb(:filtering, locals: {which:, placeholder_key:, label_key:})
     end
 
     def filter_link(jid, within = "retries")
@@ -256,14 +256,6 @@ module Sidekiq
       end
     end
 
-    def busy_weights(capsule_weights)
-      # backwards compat with 7.0.0, remove in 7.1
-      cw = [capsule_weights].flatten
-      cw.map { |hash|
-        hash.map { |name, weight| (weight > 0) ? +name << ": " << weight.to_s : name }.join(", ")
-      }.join("; ")
-    end
-
     def stats
       @stats ||= Sidekiq::Stats.new
     end
@@ -337,7 +329,7 @@ module Sidekiq
     end
 
     def csrf_tag
-      "<input type='hidden' name='authenticity_token' value='#{env[:csrf_token]}'/>"
+      ""
     end
 
     def csp_nonce