sidekiq 7.3.9 → 8.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dccc402e8f4e309a47a2adb62418fa29989012e8c94f34aa11200b098dc375f2
-  data.tar.gz: 71b64a582d5971b045d60bae38dc26da265b60ff9a099ca3b2f9c7df9035e805
+  metadata.gz: e02b7ba4c390eb2a743bc62a8938cedd64ee940a6e128b2c517d9c3f27649543
+  data.tar.gz: 86b1e8925322fb00b3d443c6aa8762d7d06f9a564276bf1b210c0415c8d571b4
 SHA512:
-  metadata.gz: d9148b613a222ca9617ceebe25bb04a5d086edacdafad4a426b6232662d3b583db462dc5e0491297f7859037c5166bcb541bddcd3949f6d3b5a1c2e3fc572b65
-  data.tar.gz: 93a797cefd1a68adb236c7538c28571467c328d6965af0bc5ce505a1391d6f5b6e2ae2c87b42110e837dcfa8d788aff55309806bbb1bcd181e205c3bc66adc29
+  metadata.gz: 63fd4147cf1a9ed566f47266a0c88eb966756285f4404714f2487783a1b754604ab00d5d9b4514002a116598f6005776c6db305c8d66192ec5c8e9b84af71595
+  data.tar.gz: a904f9f6963083f76f5645990f068f31b5f3a1a13376d73744b693d6f2e0e2d551be420d70b9c6f97d3ae3ae6691140d21af6357e208e275228ba0061e92df14

data/Changes.md

@@ -2,10 +2,235 @@
 
 [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
 
+8.1.7
+----------
+
+- Forward compatibililty with Active Job 8.1 [#7019]
+- Many minor fixes and test improvements [hammadxcm]
+
+8.1.6
+----------
+
+- Fix reported thread/memory leak when jobs fail [#7006]
+- Users can limit data displayed on Busy page with the `only` parameter `/busy?only=(jobs|processes)` [#6992]
+- Replace Rack::Utils usage with standard library APIs
+- Several minor fixes from AI scanners.
+
+8.1.5
+----------
+
+- Fix sub-second precision when computing the `retry_for` deadline [#7003]
+- Identify Sidekiq connnections in Redis with `CLIENT SETINFO` [#6986]
+- Fix edge case where Web UI could show an empty Batch set [#6987]
+
+8.1.4
+----------
+
+- The TTIN signal is undeprecated as the INFO signal is not supported on Linux
+- Show iteration job state on Busy page [#6978]
+
+8.1.3
+----------
+
+- Fix edge case leading to duplicate, concurrent execution [#6379]
+  If 2 Capsules process jobs from the same queue, long-running
+  jobs could run in parallel during process shutdown.
+- [SECURITY] Remove as much YAML usage as possible. [#6950]
+  Localization files in `web/locales` are now manually parsed.
+  Sidekiq::CLI will now only require YAML if you use a `-C` .yml file.
+
+8.1.2
+----------
+
+- Initial release for `kiq`, Sidekiq's official terminal UI:
+```
+bundle exec kiq
+```
+Use REDIS_URL or REDIS_PROVIDER to point `kiq` to Redis.
+- Mutation during iteration in `SortedSet#each` caused it to miss half of the jobs [#6936]
+- Fix edge case resulting in nil crash on /busy page [#6954]
+
+8.1.1
+----------
+
+- **DEPRECATION** `require 'sidekiq/testing'` and
+  `require 'sidekiq/testing/inline'`.
+  Add new `Sidekiq.testing!(mode)` API [#6931]
+  Requiring code should not enable process-wide changes.
+```ruby
+# Old, implicit
+require "sidekiq/testing"
+require "sidekiq/testing/inline"
+# New, more explicit
+require "sidekiq"
+Sidekiq.testing!(:fake)
+Sidekiq.testing!(:inline)
+```
+- Fix race condition with Stop button in UI [#6935]
+- Fix javascript error handler [#6893]
+
+8.1.0
+----------
+
+- `retry_for` and `retry` are now mutually exclusive [#6878, Saidbek]
+- `perform_inline` now enforces `strict_args!` [#6718, Saidbek]
+- Integrate Herb linting for ERB templates [#6760, Saidbek]
+- Remove CSRF code, use `Sec-Fetch-Site` header [#6874, deve1212]
+- Allow custom Web UI `assets_path` for CDN purposes [#6865, stanhu]
+- Upgrade to connection_pool 3.0
+- Allow idle connection reaping after N seconds.
+You can activate this **beta** feature like below.
+Feedback requested: is this feature stable and useful for you in production?
+This feature may or may not be enabled by default in Sidekiq 9.0.
+```ruby
+Sidekiq.configure_server do |cfg|
+  cfg.reap_idle_redis_connections(60)
+end
+```
+
+8.0.10
+----------
+
+- Add confirm dialog for Delete All buttons in Web UI [#6853]
+- Adjust scheduler to run closer to poll average [#6866]
+- Forward compatibility changes for connection_pool 3.0.0
+- Backwards compatibility fix for <8.0.9 process data in Redis [#6870]
+- Backtrace dump can now be triggered with the INFO signal, since Puma uses the
+  same signal [#6857]
+
+8.0.9
+----------
+
+- Implement idle Redis connection reaping, will be activated in 8.1 [#6663]
+- Updated `Sidekiq::Process` API to provide capsule data. The `queues` and `weights` 
+  data will be removed from Redis in Sidekiq 8.1, as this data can now be found in the
+  `capsules` element. [#6295]
+- Restore bulk action buttons on Scheduled, Retry and Dead tabs [#6833, deve1212]
+- Support logging additional job attributes [#6846, bschrag620]
+- Fix display of long job args [#6836]
+- Create development lifecycle (`docs/sdlc.md`) and security (`docs/SECURITY.md`) policy
+  documentation for Sidekiq's current workflows
+
+8.0.8
+----------
+
+- Allow an optional global iteration max runtime. After executing for this length of time,
+  Sidekiq will re-queue the job to continue execution at a later time [#6819, fatkodima]
+```ruby
+Sidekiq.configure_server do |cfg|
+  cfg[:max_iteration_runtime] = 600 # ten minutes
+end
+```
+- Add `discarded_at` attribute when discarding a job so death handlers can distinguish between
+  a job which was killed and one that was discarded. [#6820, gstokkink]
+- `perform_bulk` now accepts an `:at` array of times to schedule each job at the corresponding time.
+  `perform_bulk(args: [[1], [2]], at: [Time.now, Time.now + 1])` [#6790, fatkodima]
+- `perform_bulk` now accepts a `:spread_interval` value to schedule jobs over
+  the next N seconds. `perform_bulk(..., spread_interval: 60)` [#6792, fatkodima]
+- Fix unintended display of flash messages in the Web UI due to session key collision
+- Add support for lazy load hooks [#6825]
+
+8.0.7
+----------
+
+- The `:discard` option for `sidekiq_retries_exhausted` and `sidekiq_retry_in`
+  now calls death handlers, otherwise it could break other Sidekiq
+  functionality. [#6741]
+- Provide a Plain log formatter which does not colorize output [#6778]
+- Job iteration now exposes `current_object` for easy access within the `around_iteration` callback [#6774]
+- Fix JS race condition which could skip confirmation dialogs when Live Polling [#6768]
+- Fix edge case which could lose CurrentAttributes [#6767]
+- Update UK locale [#6776]
+
+8.0.6
+----------
+
+- Adjust transactional client to use ActiveRecord 7.2's support for
+  `after_all_transactions_commit` when available. [#6765, rewritten]
+- Fix Rails 7.0 and 7.1 compatibility [#6746, mlarraz]
+- Flush metrics at `:exit` [#6764]
+
+8.0.5
+----------
+
+- Add `stopping?` method to AJ adapter for compatibility with the new AJ::Continuations feature [#6732]
+- Further improvements to Rails boot compatibility [#6710]
+- Add ability to disable CSRF middleware. SameSite cookies prevent
+  CSRF in a cleaner manner and are default in most browsers now.
+  CSRF code will be removed in Sidekiq 9.0. [#6739]
+
+8.0.4
+----------
+
+- Adjust Rails integration for various edge cases [6713]
+- Flush job iteration state when an error is raised [#6704]
+- Update Accept-Language parsing in Web UI [#6721]
+- Remove fixed-width in Web UI [#6686]
+- Adjust CSRF middleware ordering [#6688]
+- Support proxies when POSTing profiles to profiler.firefox.com [#6687]
+- Dont swallow NoMethodErrors in CurrentAttributes [#6685]
+
+8.0.3
+----------
+
+- Configure Vernier output directory [#6674]
+- Rework Rails integration [#6669]
+- Implement flash messages for the Web UI [#6675]
+
+8.0.2
+----------
+
+- Add `on(:exit)` event to run code right before the Sidekiq process exits [#6637]
+- Metrics page crashes with Rack 3.1+ [#6646]
+
+8.0.1
+----------
+
+- Relax Redis requirement to 7.0 for compatibility with AWS and Ubuntu 24.04 LTS. [#6630]
+
+8.0.0
+----------
+
+- **WARNING** The underlying class name for Active Jobs has changed from `ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper` to `Sidekiq::ActiveJob::Wrapper`.
+  The old name will still work in 8.x.
+- **WARNING** The `created_at`, `enqueued_at`, `failed_at` and `retried_at` attributes are now stored as epoch milliseconds, rather than epoch floats.
+  This is meant to avoid precision issues with JSON and JavaScript's 53-bit Floats.
+  Example: `"created_at" => 1234567890.123456` -> `"created_at" => 1234567890123`.
+- **NEW FEATURE** Job Profiling is now supported with [Vernier](https://vernier.prof)
+  which makes it really easy to performance tune your slow jobs.
+  The Web UI contains a new **Profiles** tab to view any collected profile data.
+  Please read the new [Profiling](https://github.com/sidekiq/sidekiq/wiki/Profiling) wiki page for details.
+- **NEW FEATURE** Job Metrics now store up to 72 hours of data and the Web UI allows display of 24/48/72 hours. [#6614]
+- CurrentAttribute support now uses `ActiveJob::Arguments` to serialize the context object, supporting Symbols and GlobalID.
+  The change should be backwards compatible. [#6510]
+- Freshen up `Sidekiq::Web` to simplify the code and improve security [#6532]
+  The CSS has been rewritten from scratch to remove the Bootstrap framework.
+- Add `on_cancel` callback for iterable jobs [#6607]
+- Add `cursor` reader to get the current cursor inside iterable jobs [#6606]
+- Default error logging has been modified to use Ruby's `Exception#detailed_message` and `#full_message` APIs.
+- CI now runs against Redis, Dragonfly and Valkey.
+- Job tags now allow custom CSS display [#6595]
+- The Web UI's language picker now shows options in the native language
+- Remove global variable usage within the codebase
+- Colorize and adjust logging for easier reading
+- Adjust Sidekiq's default thread priority to -1 for a 50ms timeslice.
+  This can help avoid TimeoutErrors when Sidekiq is overloaded. [#6543]
+- Use `Logger#with_level`, remove Sidekiq's custom impl
+- Remove `base64` gem dependency
+- Support: (Dragonfly 1.27+, Valkey 7.2+, Redis 7.2+), Ruby 3.2+, Rails 7.0+
+
+7.3.10
+----------
+
+- Deprecate Redis :password as a String to avoid log disclosure. [#6625]
+  Use a Proc instead: `config.redis = { password: ->(username) { "password" } }`
+
 7.3.9
 ----------
 
 - Only require activejob if necessary [#6584]
+  You might get `uninitialized constant Sidekiq::ActiveJob` if you
+  `require 'sidekiq'` before `require 'rails'`.
 - Fix iterable job cancellation [#6589]
 - Web UI accessibility improvements [#6604]
 

data/README.md

@@ -3,22 +3,25 @@ Sidekiq
 
 [![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
 ![Build](https://github.com/sidekiq/sidekiq/workflows/CI/badge.svg)
+[![Ruby Users Forum](https://img.shields.io/discourse/topics?server=https%3A%2F%2Fwww.rubyforum.org&style=flat&logo=discourse&label=Ruby%20Users%20Forum)](https://www.rubyforum.org/tag/sidekiq)
 
-Simple, efficient background processing for Ruby.
+Simple, efficient background jobs for Ruby.
 
 Sidekiq uses threads to handle many jobs at the same time in the
-same process.  It does not require Rails but will integrate tightly with
-Rails to make background processing dead simple.
+same process. Sidekiq can be used by any Ruby application.
 
 
 Requirements
 -----------------
 
-- Redis: Redis 6.2+ or Dragonfly 1.13+
-- Ruby: MRI 2.7+ or JRuby 9.3+.
+- Redis: Redis 7.0+, Valkey 7.2+ or Dragonfly 1.27+
+- Ruby: MRI 3.2+ or JRuby 9.4+.
 
-Sidekiq 7.0 supports Rails 6.0+ but does not require it.
-As of 7.2, Sidekiq supports Dragonfly as an alternative to Redis for data storage.
+Sidekiq 8.0 supports Rails and Active Job 7.0+.
+
+Sidekiq supports [Valkey](https://valkey.io) and [Dragonfly](https://www.dragonflydb.io) as Redis alternatives.
+Redis 7.2.4 is considered to be the canonical implementation.
+Incompatibilities with that version are considered bugs.
 
 Installation
 -----------------
@@ -42,6 +45,7 @@ The benchmark in `bin/sidekiqload` creates 500,000 no-op jobs and drains them as
 This requires a lot of Redis network I/O and JSON parsing.
 This benchmark is IO-bound so we increase the concurrency to 25.
 If your application is sending lots of emails or performing other network-intensive work, you could see a similar benefit but be careful not to saturate the CPU.
+Real world applications will rarely if ever need to use concurrency greater than 10.
 
 Version | Time to process 500k jobs | Throughput (jobs/sec) | Ruby | Concurrency | Job Type
 -----------------|------|---------|---------|------------------------|---
@@ -62,7 +66,7 @@ Want to Upgrade?
 Use `bundle up sidekiq` to upgrade Sidekiq and all its dependencies.
 Upgrade notes between each major version can be found in the `docs/` directory.
 
-I also sell Sidekiq Pro and Sidekiq Enterprise, extensions to Sidekiq which provide more
+I also sell [Sidekiq Pro](https://billing.contribsys.com/spro/) and [Sidekiq Enterprise](https://billing.contribsys.com/sent/new.cgi), extensions to Sidekiq which provide more
 features, a commercial-friendly license and allow you to support high
 quality open source development all at the same time.  Please see the
 [Sidekiq](https://sidekiq.org/) homepage for more detail.
@@ -71,7 +75,7 @@ quality open source development all at the same time.  Please see the
 Problems?
 -----------------
 
-**Please do not directly email any Sidekiq committers with questions or problems.**
+**Do not directly email any Sidekiq committers with questions or problems.**
 A community is best served when discussions are held in public.
 
 If you have a problem, please review the [FAQ](https://github.com/sidekiq/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/sidekiq/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages.
@@ -85,22 +89,38 @@ Useful resources:
 * Product documentation is in the [wiki](https://github.com/sidekiq/sidekiq/wiki).
 * Occasional announcements are made to the [@sidekiq](https://ruby.social/@sidekiq) Mastodon account.
 * The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A.
+* Join us in the `sidekiq` tag on the [Ruby Users Forum](https://www.rubyforum.org/tag/sidekiq).
 
-Every Thursday morning is Sidekiq office hour: I video chat and answer questions.
-See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
+Every Thursday morning is Sidekiq Office Hour: I video chat and answer questions.
+See the [Sidekiq support page](https://sidekiq.org/support/) for details.
 
 Contributing
 -----------------
 
-Please see [the contributing guidelines](https://github.com/sidekiq/sidekiq/blob/main/.github/contributing.md).
+See [the contributing guidelines](https://github.com/sidekiq/sidekiq/blob/main/.github/contributing.md).
+
+### ERB Linting with HERB
+
+This project uses [HERB](https://herb-tools.dev/) for ERB file linting and formatting. All ERB files have been renamed to use the `.html.erb` extension for better tooling support.
+
+**Local Development:**
+```bash
+# Run HERB linting
+bundle exec rake lint:herb
+# or
+bin/lint-herb
+```
+
+**CI Integration:**
+HERB linting is automatically run in CI to ensure all ERB files are properly formatted and free of parse errors.
 
 License
 -----------------
 
-Please see [LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/LICENSE.txt) for licensing details.
+See [LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/LICENSE.txt) for licensing details.
 The license for Sidekiq Pro and Sidekiq Enterprise can be found in [COMM-LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/COMM-LICENSE.txt).
 
 Author
 -----------------
 
-Mike Perham, [@getajobmike](https://ruby.social/@getajobmike) / [@sidekiq](https://ruby.social/@sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
+Mike Perham, [mastodon](https://ruby.social/@getajobmike), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)

data/bin/kiq

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+# This requires the default gemset so Sidekiq Pro
+# and Sidekiq Enterprise can load any code extensions.
+Bundler.require(:default, :tui)
+
+require_relative "../lib/sidekiq/tui"
+
+# Run any load hooks registered during Bundler.require
+Sidekiq.loader.run_load_hooks(:tui)
+
+tt = Sidekiq::TUI.new(Sidekiq.default_configuration)
+
+RatatuiRuby.run do |tui|
+  tt.prepare(tui)
+  tt.run_loop
+end

data/bin/lint-herb

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# HERB Linting Script
+# Run this script to lint all ERB files in the project
+# Usage: bin/lint-herb
+
+require "bundler/setup"
+
+puts "🔍 Running HERB linting on ERB files..."
+puts
+
+exec("bundle exec herb analyze web/views -n --no-log-file")

data/bin/sidekiqload

@@ -1,5 +1,7 @@
 #!/usr/bin/env ruby
 
+ENV["RUBY_TCP_NO_FAST_FALLBACK"] = "1"
+
 #
 # bin/sidekiqload is a helpful script to load test and
 # performance tune Sidekiq's core. It creates 500,000 no-op
@@ -28,7 +30,7 @@ require "ruby-prof" if ENV["PROFILE"]
 require "bundler/setup"
 Bundler.require(:default, :load_test)
 
-latency = Integer(ENV["LATENCY"] || 1)
+latency = Integer(ENV["LATENCY"] || 0)
 if latency > 0
   # brew tap shopify/shopify
   # brew install toxiproxy
@@ -86,7 +88,7 @@ class Loader
   def initialize
     @iter = ENV["GC"] ? 10 : 500
     @count = Integer(ENV["COUNT"] || 1_000)
-    @latency = Integer(ENV["LATENCY"] || 1)
+    @latency = Integer(ENV["LATENCY"] || 0)
   end
 
   def configure
@@ -137,7 +139,7 @@ class Loader
   end
 
   def setup
-    Sidekiq.logger.error("Setup RSS: #{Process.rss}")
+    Sidekiq.logger.warn("Setup RSS: #{Process.rss}")
     Sidekiq.redis { |c| c.flushdb }
     start = Time.now
     if ENV["AJ"]
@@ -167,9 +169,9 @@ class Loader
         if total == 0
           ending = Time.now - @start
           size = @iter * @count
-          Sidekiq.logger.error("Done, #{size} jobs in #{ending} sec, #{(size / ending).to_i} jobs/sec")
-          Sidekiq.logger.error("Ending RSS: #{Process.rss}")
-          Sidekiq.logger.error("Now here's the latency for three jobs")
+          Sidekiq.logger.warn("Done, #{size} jobs in #{ending} sec, #{(size / ending).to_i} jobs/sec")
+          Sidekiq.logger.warn("Ending RSS: #{Process.rss}")
+          Sidekiq.logger.warn("Now here's the latency for three jobs")
 
           if ENV["AJ"]
             LoadJob.perform_later("", 1, {}, Time.now.to_f)
@@ -191,7 +193,7 @@ class Loader
   end
 
   def with_latency(latency, &block)
-    Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
+    Sidekiq.logger.warn "Simulating #{latency}ms of latency between Sidekiq and redis"
     if latency > 0
       Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
     else
@@ -204,8 +206,8 @@ class Loader
     monitor
 
     if ENV["PROFILE"]
-      RubyProf.exclude_threads = [@monitor]
-      RubyProf.start
+      $profile = RubyProf::Profile.new(exclude_threads: [@monitor])
+      $profile.start
     elsif ENV["GC"]
       GC.start
       GC.compact
@@ -236,7 +238,7 @@ class Loader
     Sidekiq.logger.error("GC End RSS: #{Process.rss}") if ENV["GC"]
     if ENV["PROFILE"]
       Sidekiq.logger.error("Profiling...")
-      result = RubyProf.stop
+      result = $profile.stop
       printer = RubyProf::GraphHtmlPrinter.new(result)
       printer.print(File.new("output.html", "w"), min_percent: 1)
     end

data/bin/webload

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+require 'bundler'
+Bundler.setup
+
+# This skeleton allows you to run Sidekiq::Web page rendering
+# through Vernier for tuning.
+require "sidekiq/web"
+require "rack/test"
+require "vernier"
+
+Sidekiq::Web.configure do |config|
+  config.middlewares.clear # remove csrf
+end
+
+class SomeJob
+  include Sidekiq::Job
+end
+
+class BenchWeb
+  include Rack::Test::Methods
+
+  def app
+    Sidekiq::Web.new
+  end
+
+  def warmup(page = "/scheduled")
+    # Sidekiq.redis {|c| c.flushdb }
+
+    # 100.times do |idx|
+    #   SomeJob.perform_at(idx, 1, 3, "mike", {"foo" => "bar"})
+    # end
+
+    100.times do
+      get page
+    end
+  end
+
+  def load(page = "/scheduled", count = 10_000)
+    profile do
+      count.times do
+        get page
+        raise last_response.inspect unless last_response.status == 200
+      end
+    end
+  end
+
+  def profile(&)
+    if ENV["PROF"]
+      Vernier.profile(out: "profile.json.gz", &)
+    else
+      yield
+    end
+  end
+end
+
+def timer(name="block", count = 10_000)
+  a = Time.now
+  yield count
+  b = Time.now
+  puts "#{name} in #{b - a} sec"
+end
+
+page = "/busy"
+b = BenchWeb.new
+b.warmup(page)
+timer(page) do |count|
+  b.load(page, count)
+end