sidekiq 6.0.0 → 6.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fda3ed37e1d981e1b32eea70669c678d400c071a9146a919848bcc598612de25
-  data.tar.gz: 6580ab6c188b2514efe807479ca25c06ea584d1def175d84d94639bc20f80b1e
+  metadata.gz: b72b92d3d281aaea2b4781cb7743d95a8c7f42708526383bf85fbed420b4b82c
+  data.tar.gz: 9546107658b47c8f68497730e2ee6689881c1388ba96db3f2415b0fda7b7f668
 SHA512:
-  metadata.gz: 25682d3dbf99d29fa3f56b7c74419863d3549b6e0c182e69c83fbc02a2955d053bca85eec53b474d4d22c7526e44e865f378aa4111152a91377b373be27c4d47
-  data.tar.gz: e55d07434b1e2ab26df6748101bf10640fc08d4d93f00b75ed04d82ed309ee8529d652982430cd00671c98b5e53cc733bef4b686df58a6b0e116ba04c12139aa
+  metadata.gz: 78f0af1f4ed41c5b4ad01636d9808d409d1cedc0bc6146be278412e87f9d826fad2cec9c89b31acbc6da9bab59bff44e826418991b10ce27b34631bcc3a00cf2
+  data.tar.gz: a586b9ec613cf3da47ecb77eb4e0f5d6e497f234369d958cdfa3d8c8c7418806383e022a9b08db4c3b5d08a1c2aac3aaecdf851744234f4c42055e1a7a6ebf6c

data/Changes.md

@@ -1,13 +1,250 @@
 # Sidekiq Changes
 
-[Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
+[Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
+
+HEAD
+---------
+
+- **BREAK**: The Web UI has been refactored to remove jQuery. Any UI extensions
+  which use jQuery will break.
+- **FEATURE**: Sidekiq.logger has been enhanced so any `Rails.logger`
+  output in jobs now shows up in the Sidekiq console. Remove any logger
+  hacks in your initializer and see if it Just Works™ now. [#5021]
+- **FEATURE**: Add `Sidekiq::Job` alias for `Sidekiq::Worker`, to better
+  reflect industry standard terminology. You can now do this:
+```ruby
+class MyJob
+  include Sidekiq::Job
+  sidekiq_options ...
+  def perform(args)
+  end
+end
+```
+- **FEATURE**: Support for serializing ActiveSupport::CurrentAttributes into each job. [#4982]
+```ruby
+# config/initializers/sidekiq.rb
+require "sidekiq/middleware/current_attributes"
+Sidekiq::CurrentAttributes.persist(Myapp::Current) # Your AS::CurrentAttributes singleton
+```
+- **FEATURE**: Add `Sidekiq::Worker.perform_bulk` for enqueuing jobs in bulk,
+  similar to `Sidekiq::Client.push_bulk` [#5042]
+```ruby
+MyJob.perform_bulk([[1], [2], [3]])
+```
+- Implement `queue_as`, `wait` and `wait_until` for ActiveJob compatibility [#5003]
+- Scheduler now uses Lua to reduce Redis load and network roundtrips [#5044]
+- Retry Redis operation if we get an `UNBLOCKED` Redis error [#4985]
+- Run existing signal traps, if any, before running Sidekiq's trap [#4991]
+- Fix fetch bug when using weighted queues which caused Sidekiq to stop
+  processing queues randomly [#5031]
+
+6.2.2
+---------
+
+- Reduce retry jitter, add jitter to `sidekiq_retry_in` values [#4957]
+- Minimize scheduler load on Redis at scale [#4882]
+- Improve logging of delay jobs [#4904, BuonOno]
+- Minor CSS improvements for buttons and tables, design PRs always welcome!
+- Tweak Web UI `Cache-Control` header [#4966]
+- Rename internal API class `Sidekiq::Job` to `Sidekiq::JobRecord` [#4955]
+
+6.2.1
+---------
+
+- Update RTT warning logic to handle transient RTT spikes [#4851]
+- Fix very low priority CVE on unescaped queue name [#4852]
+- Add note about sessions and Rails apps in API mode
+
+6.2.0
+---------
+
+- Store Redis RTT and log if poor [#4824]
+- Add process/thread stats to Busy page [#4806]
+- Improve Web UI on mobile devices [#4840]
+- **Refactor Web UI session usage** [#4804]
+  Numerous people have hit "Forbidden" errors and struggled with Sidekiq's
+  Web UI session requirement. If you have code in your initializer for
+  Web sessions, it's quite possible it will need to be removed. Here's
+  an overview:
+```
+Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
+make sure you mount Sidekiq::Web *inside* your routes in `config/routes.rb` so
+Sidekiq can reuse the Rails session:
+
+  Rails.application.routes.draw do
+    mount Sidekiq::Web => "/sidekiq"
+    ....
+  end
+
+If this is a bare Rack app, use a session middleware before Sidekiq::Web:
+
+  # first, use IRB to create a shared secret key for sessions and commit it
+  require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
+
+  # now, update your Rack app to include the secret with a session cookie middleware
+  use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
+  run Sidekiq::Web
+
+If this is a Rails app in API mode, you need to enable sessions.
+
+  https://guides.rubyonrails.org/api_app.html#using-session-middlewares
+```
+
+6.1.3
+---------
+
+- Warn if Redis is configured to evict data under memory pressure [#4752]
+- Add process RSS on the Busy page [#4717]
+
+6.1.2
+---------
+
+- Improve readability in dark mode Web UI [#4674]
+- Fix Web UI crash with corrupt session [#4672]
+- Allow middleware to yield arguments [#4673, @eugeneius]
+- Migrate CI from CircleCI to GitHub Actions [#4677]
+
+6.1.1
+---------
+
+- Jobs are now sorted by age in the Busy Workers table. [#4641]
+- Fix "check all" JS logic in Web UI [#4619]
+
+6.1.0
+---------
+
+- Web UI - Dark Mode fixes [#4543, natematykiewicz]
+- Ensure `Rack::ContentLength` is loaded as middleware for correct Web UI responses [#4541]
+- Avoid exception dumping SSL store in Redis connection logging [#4532]
+- Better error messages in Sidekiq::Client [#4549]
+- Remove rack-protection, reimplement CSRF protection [#4588]
+- Require redis-rb 4.2 [#4591]
+- Update to jquery 1.12.4 [#4593]
+- Refactor internal fetch logic and API [#4602]
+
+6.0.7
+---------
+
+- Refactor systemd integration to work better with custom binaries [#4511]
+- Don't connect to Redis at process exit if not needed [#4502]
+- Remove Redis connection naming [#4479]
+- Fix Redis Sentinel password redaction [#4499]
+- Add Vietnamese locale (vi) [#4528]
+
+6.0.6
+---------
+
+- **Integrate with systemd's watchdog and notification features** [#4488]
+  Set `Type=notify` in [sidekiq.service](https://github.com/mperham/sidekiq/blob/4b8a8bd3ae42f6e48ae1fdaf95ed7d7af18ed8bb/examples/systemd/sidekiq.service#L30-L39). The integration works automatically.
+- Use `setTimeout` rather than `setInterval` to avoid thundering herd [#4480]
+- Fix edge case where a job can be pushed without a queue.
+- Flush job stats at exit [#4498]
+- Check RAILS_ENV before RACK_ENV [#4493]
+- Add Lithuanian locale [#4476]
+
+6.0.5
+---------
+
+- Fix broken Web UI response when using NewRelic and Rack 2.1.2+. [#4440]
+- Update APIs to use `UNLINK`, not `DEL`. [#4449]
+- Fix Ruby 2.7 warnings [#4412]
+- Add support for `APP_ENV` [[95fa5d9]](https://github.com/mperham/sidekiq/commit/95fa5d90192148026e52ca2902f1b83c70858ce8)
+
+6.0.4
+---------
+
+- Fix ActiveJob's `sidekiq_options` integration [#4404]
+- Sidekiq Pro users will now see a Pause button next to each queue in
+  the Web UI, allowing them to pause queues manually [#4374, shayonj]
+- Fix Sidekiq::Workers API unintentional change in 6.0.2 [#4387]
+
+
+6.0.3
+---------
+
+- Fix `Sidekiq::Client.push_bulk` API which was erroneously putting
+  invalid `at` values in the job payloads [#4321]
+
+6.0.2
+---------
+
+- Fix Sidekiq Enterprise's rolling restart functionality, broken by refactoring in 6.0.0. [#4334]
+- More internal refactoring and performance tuning [fatkodima]
+
+6.0.1
+---------
+
+- **Performance tuning**, Sidekiq should be 10-15% faster now [#4303, 4299,
+  4269, fatkodima]
+- **Dark Mode support in Web UI** (further design polish welcome!) [#4227, mperham,
+  fatkodima, silent-e]
+- **Job-specific log levels**, allowing you to turn on debugging for
+  problematic workers. [fatkodima, #4287]
+```ruby
+MyWorker.set(log_level: :debug).perform_async(...)
+```
+- **Ad-hoc job tags**. You can tag your jobs with, e.g, subdomain, tenant, country,
+  locale, application, version, user/client, "alpha/beta/pro/ent", types of jobs,
+  teams/people responsible for jobs, additional metadata, etc.
+  Tags are shown on different pages with job listings. Sidekiq Pro users
+  can filter based on them [fatkodima, #4280]
+```ruby
+class MyWorker
+  include Sidekiq::Worker
+  sidekiq_options tags: ['bank-ops', 'alpha']
+  ...
+end
+```
+- Fetch scheduled jobs in batches before pushing into specific queues.
+  This will decrease enqueueing time of scheduled jobs by a third. [fatkodima, #4273]
+```
+ScheduledSet with 10,000 jobs
+Before: 56.6 seconds
+After:  39.2 seconds
+```
+- Compress error backtraces before pushing into Redis, if you are
+  storing error backtraces, this will halve the size of your RetrySet
+  in Redis [fatkodima, #4272]
+```
+RetrySet with 100,000 jobs
+Before: 261 MB
+After:  129 MB
+```
+- Support display of ActiveJob 6.0 payloads in the Web UI [#4263]
+- Add `SortedSet#scan` for pattern based scanning. For large sets this API will be **MUCH** faster
+  than standard iteration using each. [fatkodima, #4262]
+```ruby
+  Sidekiq::DeadSet.new.scan("UnreliableApi") do |job|
+    job.retry
+  end
+```
+- Dramatically speed up SortedSet#find\_job(jid) by using Redis's ZSCAN
+  support, approx 10x faster. [fatkodima, #4259]
+```
+zscan   0.179366   0.047727   0.227093 (  1.161376)
+enum    8.522311   0.419826   8.942137 (  9.785079)
+```
+- Respect rails' generators `test_framework` option and gracefully handle extra `worker` suffix on generator [fatkodima, #4256]
+- Add ability to sort 'Enqueued' page on Web UI by position in the queue [fatkodima, #4248]
+- Support `Client.push_bulk` with different delays [fatkodima, #4243]
+```ruby
+Sidekiq::Client.push_bulk("class" => FooJob, "args" => [[1], [2]], "at" => [1.minute.from_now.to_f, 5.minutes.from_now.to_f])
+```
+- Easier way to test enqueuing specific ActionMailer and ActiveRecord delayed jobs. Instead of manually
+  parsing embedded class, you can now test by fetching jobs for specific classes. [fatkodima, #4292]
+```ruby
+assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs_for(FooMailer).size
+```
+- Add `sidekiqmon` to gemspec executables [#4242]
+- Gracefully handle `Sidekiq.logger = nil` [#4240]
+- Inject Sidekiq::LogContext module if user-supplied logger does not include it [#4239]
 
 6.0
 ---------
 
 This release has major breaking changes.  Read and test carefully in production.
 
-- ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq
+- With Rails 6.0.2+, ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq
   features/internals like the retry subsystem. [#4213, pirj]
 ```ruby
 class MyJob < ActiveJob::Base
@@ -17,6 +254,13 @@ class MyJob < ActiveJob::Base
   end
 end
 ```
+- Logging has been redesigned to allow for pluggable log formatters:
+```ruby
+Sidekiq.configure_server do |config|
+  config.log_formatter = Sidekiq::Logger::Formatters::JSON.new
+end
+```
+See the [Logging wiki page](https://github.com/mperham/sidekiq/wiki/Logging) for more details.
 - **BREAKING CHANGE** Validate proper usage of the `REDIS_PROVIDER`
   variable.  This variable is meant to hold the name of the environment
   variable which contains your Redis URL, so that you can switch Redis
@@ -36,6 +280,18 @@ end
 - Integrate the StandardRB code formatter to ensure consistent code
   styling. [#4114, gearnode]
 
+5.2.9
+---------
+
+- Release Rack lock due to a cascade of CVEs. [#4566]
+  Pro-tip: don't lock Rack.
+
+5.2.8
+---------
+
+- Lock to Rack 2.0.x to prevent future incompatibilities
+- Fix invalid reference in `sidekiqctl`
+
 5.2.7
 ---------
 

data/LICENSE

@@ -6,4 +6,4 @@ for license text.
 
 Sidekiq Pro has a commercial-friendly license allowing private forks
 and modifications of Sidekiq.  Please see https://sidekiq.org/products/pro.html for
-more detail.  You can find the commercial license terms in COMM-LICENSE.
+more detail.  You can find the commercial license terms in COMM-LICENSE.txt.

data/README.md

@@ -2,10 +2,7 @@ Sidekiq
 ==============
 
 [![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
-[![Code Climate](https://codeclimate.com/github/mperham/sidekiq.svg)](https://codeclimate.com/github/mperham/sidekiq)
-[![Build Status](https://circleci.com/gh/mperham/sidekiq/tree/master.svg?style=svg)](https://circleci.com/gh/mperham/sidekiq/tree/master)
-[![Gitter Chat](https://badges.gitter.im/mperham/sidekiq.svg)](https://gitter.im/mperham/sidekiq)
-
+![Build](https://github.com/mperham/sidekiq/workflows/CI/badge.svg)
 
 Simple, efficient background processing for Ruby.
 
@@ -18,7 +15,8 @@ Performance
 
 Version |	Latency | Garbage created for 10k jobs	| Time to process 100k jobs |	Throughput | Ruby
 -----------------|------|---------|---------|------------------------|-----
-Sidekiq 6.0.0    | 3 ms	| 156 MB  | 19 sec  | **5200 jobs/sec** | MRI 2.6.3
+Sidekiq 6.0.2    | 3 ms	| 156 MB  | 14.0 sec| **7100 jobs/sec** | MRI 2.6.3
+Sidekiq 6.0.0    | 3 ms	| 156 MB  | 19 sec  | 5200 jobs/sec | MRI 2.6.3
 Sidekiq 4.0.0    | 10 ms	| 151 MB  | 22 sec  | 4500 jobs/sec |
 Sidekiq 3.5.1    | 22 ms	| 1257 MB | 125 sec | 800 jobs/sec |
 Resque 1.25.2    |  -	  | -       | 420 sec | 240 jobs/sec |
@@ -48,7 +46,7 @@ See the [Getting Started wiki page](https://github.com/mperham/sidekiq/wiki/Gett
 You can watch [this Youtube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
 Sidekiq and see its features in action.  Here's the Web UI:
 
-![Web UI](https://github.com/mperham/sidekiq/raw/master/examples/web-ui.png)
+![Web UI](https://github.com/mperham/sidekiq/raw/main/examples/web-ui.png)
 
 
 Want to Upgrade?
@@ -86,10 +84,10 @@ See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
 License
 -----------------
 
-Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for licensing details.
+Please see [LICENSE](https://github.com/mperham/sidekiq/blob/main/LICENSE) for licensing details.
 
 
 Author
 -----------------
 
-Mike Perham, [@mperham@mastodon.xyz](https://mastodon.xyz/@mperham) / [@sidekiq](https://twitter.com/sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
+Mike Perham, [@getajobmike](https://twitter.com/getajobmike) / [@sidekiq](https://twitter.com/sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)

data/bin/sidekiq

@@ -6,13 +6,37 @@ $TESTING = false
 
 require_relative '../lib/sidekiq/cli'
 
+def integrate_with_systemd
+  return unless ENV["NOTIFY_SOCKET"]
+
+  Sidekiq.configure_server do |config|
+    Sidekiq.logger.info "Enabling systemd notification integration"
+    require "sidekiq/sd_notify"
+    config.on(:startup) do
+      Sidekiq::SdNotify.ready
+    end
+    config.on(:shutdown) do
+      Sidekiq::SdNotify.stopping
+    end
+    Sidekiq.start_watchdog if Sidekiq::SdNotify.watchdog?
+  end
+end
+
 begin
   cli = Sidekiq::CLI.instance
   cli.parse
+
+  integrate_with_systemd
+
   cli.run
 rescue => e
   raise e if $DEBUG
-  STDERR.puts e.message
-  STDERR.puts e.backtrace.join("\n")
+  if Sidekiq.error_handlers.length == 0
+    STDERR.puts e.message
+    STDERR.puts e.backtrace.join("\n")
+  else
+    cli.handle_exception e
+  end
+
   exit 1
 end

data/bin/sidekiqload

@@ -5,7 +5,8 @@
 $TESTING = false
 
 #require 'ruby-prof'
-Bundler.require(:default)
+require 'bundler/setup'
+Bundler.require(:default, :load_test)
 
 require_relative '../lib/sidekiq/cli'
 require_relative '../lib/sidekiq/launcher'
@@ -102,17 +103,20 @@ iter.times do
 end
 Sidekiq.logger.error "Created #{count*iter} jobs"
 
+start = Time.now
+
 Monitoring = Thread.new do
   watchdog("monitor thread") do
     while true
-      sleep 0.5
+      sleep 0.2
       qsize = Sidekiq.redis do |conn|
         conn.llen "queue:default"
       end
       total = qsize
-      Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
+      #Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
       if total == 0
-        Sidekiq.logger.error("Done, now here's the latency for three jobs")
+        Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
+        Sidekiq.logger.error("Now here's the latency for three jobs")
 
         LoadWorker.perform_async(1, Time.now.to_f)
         LoadWorker.perform_async(2, Time.now.to_f)

data/bin/sidekiqmon

@@ -2,8 +2,7 @@
 
 require 'sidekiq/monitor'
 
-if ARGV[0] == 'status'
-  Sidekiq::Monitor::Status.new.display(ARGV[1])
-else
-  Sidekiq::Monitor.print_usage
-end
+section = "all"
+section = ARGV[0] if ARGV.size == 1
+
+Sidekiq::Monitor::Status.new.display(section)

data/lib/generators/sidekiq/worker_generator.rb

@@ -16,7 +16,9 @@ module Sidekiq
       end
 
       def create_test_file
-        if defined?(RSpec)
+        return unless test_framework
+
+        if test_framework == :rspec
           create_worker_spec
         else
           create_worker_test
@@ -42,6 +44,14 @@ module Sidekiq
         )
         template "worker_test.rb.erb", template_file
       end
+
+      def file_name
+        @_file_name ||= super.sub(/_?worker\z/i, "")
+      end
+
+      def test_framework
+        ::Rails.application.config.generators.options[:rails][:test_framework]
+      end
     end
   end
 end