sidekiq 6.4.0 → 6.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9622b2851203b0c5a80695ab7801ca77e15dc63a641ae79132cda9a2fcbe0cc6
-  data.tar.gz: dd943a02d2cf910f51866d02254f3a7845cb159735bd54d223dd7127a1fbd2fa
+  metadata.gz: 1afbc6a1a0b14403e9148e746c08e0a2b24e634fca05288982c96719675607de
+  data.tar.gz: 3ff3f8df76b565f42030462eb8d09b673b89751dcd6a0b7c41a255789960d321
 SHA512:
-  metadata.gz: 65bcd542866d8699ecf5958d81a15fd322cd1c51dfc1dbc6a8b15402b70862510c134e2b0ed9f9dbcdbb4dea3a59c1d12878ff926145e503079a59896f279ff3
-  data.tar.gz: 36143e85dc7fd4611f8a43fe39788dc590725ac63a827fbc174916da2d59fdadb3fb64fa0d819c4ef90694f6f292ea15f7fb58fb29f368c0a06c5b90f1b4bca7
+  metadata.gz: e8a68611735322d98cc517f1d03ef02394497f8eb505e0db496909cac1f6b7f0179f38ce1966546f717115395102c78fdad9ae0fd947360307b9288dcc22b369
+  data.tar.gz: 163e41dfb153a4e2ec50d407bde08cb8a395909807053f24478a73af1a6a9858b5aec92cbf618d4d066560a2b7bedaa2f88b48f823609e2d78f21320270cb97e

data/Changes.md

@@ -2,6 +2,53 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
 
+6.5.1
+----------
+
+- Fix `push_bulk` breakage [#5387]
+
+6.5.0
+---------
+
+- Substantial refactoring of Sidekiq server internals, part of a larger effort
+  to reduce Sidekiq's internal usage of global methods and data, see [docs/global_to_local.md](docs/global_to_local.md) and [docs/middleware.md](docs/middleware.md).
+- **Add beta support for the `redis-client` gem**. This will become the default Redis driver in Sidekiq 7.0. [#5298]
+  Read more: https://github.com/mperham/sidekiq/wiki/Using-redis-client
+- **Add beta support for DB transaction-aware client** [#5291]
+  Add this line to your initializer and any jobs created during a transaction
+  will only be pushed to Redis **after the transaction commits**. You will need to add the
+  `after_commit_everywhere` gem to your Gemfile.
+```ruby
+Sidekiq.transactional_push!
+```
+  This feature does not have a lot of production usage yet; please try it out and let us
+  know if you have any issues. It will be fully supported in Sidekiq 7.0 or removed if it
+  proves problematic.
+- Fix regression with middleware arguments [#5312]
+
+6.4.2
+---------
+
+- Strict argument checking now runs after client-side middleware [#5246]
+- Fix page events with live polling [#5184]
+- Many under-the-hood changes to remove all usage of the term "worker"
+  from the Sidekiq codebase and APIs. This mostly involved RDoc and local
+  variable names but a few constants and public APIs were changed. The old
+  APIs will be removed in Sidekiq 7.0.
+```
+Sidekiq::DEFAULT_WORKER_OPTIONS -> Sidekiq.default_job_options
+Sidekiq.default_worker_options -> Sidekiq.default_job_options
+Sidekiq::Queues["default"].jobs_by_worker(HardJob) -> Sidekiq::Queues["default"].jobs_by_class(HardJob)
+```
+
+6.4.1
+---------
+
+- Fix pipeline/multi deprecations in redis-rb 4.6
+- Fix sidekiq.yml YAML load errors on Ruby 3.1 [#5141]
+- Sharding support for `perform_bulk` [#5129]
+- Refactor job logger for SPEEEEEEED
+
 6.4.0
 ---------
 
@@ -27,7 +74,6 @@ bin/rails generate sidekiq:job ProcessOrderJob
 ```
 - Fix job retries losing CurrentAttributes [#5090]
 - Tweak shutdown to give long-running threads time to cleanup [#5095]
-- Add keyword arguments support in extensions
 
 6.3.1
 ---------
@@ -312,6 +358,13 @@ See the [Logging wiki page](https://github.com/mperham/sidekiq/wiki/Logging) for
 - Integrate the StandardRB code formatter to ensure consistent code
   styling. [#4114, gearnode]
 
+5.2.10
+---------
+
+- Backport fix for CVE-2022-23837.
+- Migrate to `exists?` for redis-rb.
+- Lock redis-rb to <4.6 to avoid deprecations.
+
 5.2.9
 ---------
 

data/README.md

@@ -36,7 +36,7 @@ Sidekiq 6.0 supports Rails 5.0+ but does not require it.
 Installation
 -----------------
 
-    gem install sidekiq
+    bundle add sidekiq
 
 
 Getting Started
@@ -80,6 +80,11 @@ Useful resources:
 Every Friday morning is Sidekiq happy hour: I video chat and answer questions.
 See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
 
+Contributing
+-----------------
+
+Please see [the contributing guidelines](https://github.com/mperham/sidekiq/blob/main/.github/contributing.md).
+
 
 License
 -----------------

data/bin/sidekiq

@@ -4,7 +4,7 @@
 # RUBYOPT=-w bundle exec sidekiq
 $TESTING = false
 
-require_relative '../lib/sidekiq/cli'
+require_relative "../lib/sidekiq/cli"
 
 def integrate_with_systemd
   return unless ENV["NOTIFY_SOCKET"]
@@ -32,8 +32,8 @@ begin
 rescue => e
   raise e if $DEBUG
   if Sidekiq.error_handlers.length == 0
-    STDERR.puts e.message
-    STDERR.puts e.backtrace.join("\n")
+    warn e.message
+    warn e.backtrace.join("\n")
   else
     cli.handle_exception e
   end

data/bin/sidekiqload

@@ -4,19 +4,22 @@
 # RUBYOPT=-w bundle exec sidekiq
 $TESTING = false
 
-#require 'ruby-prof'
-require 'bundler/setup'
+# require "ruby-prof"
+require "bundler/setup"
 Bundler.require(:default, :load_test)
 
-require_relative '../lib/sidekiq/cli'
-require_relative '../lib/sidekiq/launcher'
+require_relative "../lib/sidekiq/cli"
+require_relative "../lib/sidekiq/launcher"
 
-include Sidekiq::Util
+if ENV["SIDEKIQ_REDIS_CLIENT"]
+  Sidekiq::RedisConnection.adapter = :redis_client
+end
 
 Sidekiq.configure_server do |config|
   config.options[:concurrency] = 10
-  config.redis = { db: 13, port: 6380, driver: :hiredis }
-  config.options[:queues] << 'default'
+  config.redis = {db: 13, port: 6380}
+  # config.redis = { db: 13, port: 6380, driver: :hiredis}
+  config.options[:queues] << "default"
   config.logger.level = Logger::ERROR
   config.average_scheduled_poll_interval = 2
   config.reliable! if defined?(Sidekiq::Pro)
@@ -29,53 +32,50 @@ class LoadWorker
     1
   end
 
-  def perform(idx, ts=nil)
-    puts(Time.now.to_f - ts) if ts != nil
-    #raise idx.to_s if idx % 100 == 1
+  def perform(idx, ts = nil)
+    puts(Time.now.to_f - ts) if !ts.nil?
+    # raise idx.to_s if idx % 100 == 1
   end
 end
 
 # brew tap shopify/shopify
 # brew install toxiproxy
-# gem install toxiproxy
 # run `toxiproxy-server` in a separate terminal window.
-require 'toxiproxy'
+require "toxiproxy"
 # simulate a non-localhost network for realer-world conditions.
 # adding 1ms of network latency has an ENORMOUS impact on benchmarks
 Toxiproxy.populate([{
-    "name": "redis",
-    "listen": "127.0.0.1:6380",
-    "upstream": "127.0.0.1:6379"
+  name: "redis",
+  listen: "127.0.0.1:6380",
+  upstream: "127.0.0.1:6379"
 }])
 
 self_read, self_write = IO.pipe
-%w(INT TERM TSTP TTIN).each do |sig|
-  begin
-    trap sig do
-      self_write.puts(sig)
-    end
-  rescue ArgumentError
-    puts "Signal #{sig} not supported"
+%w[INT TERM TSTP TTIN].each do |sig|
+  trap sig do
+    self_write.puts(sig)
   end
+rescue ArgumentError
+  puts "Signal #{sig} not supported"
 end
 
-Sidekiq.redis {|c| c.flushdb}
+Sidekiq.redis { |c| c.flushdb }
 def handle_signal(launcher, sig)
   Sidekiq.logger.debug "Got #{sig} signal"
   case sig
-  when 'INT'
+  when "INT"
     # Handle Ctrl-C in JRuby like MRI
     # http://jira.codehaus.org/browse/JRUBY-4637
     raise Interrupt
-  when 'TERM'
+  when "TERM"
     # Heroku sends TERM and then waits 30 seconds for process to exit.
     raise Interrupt
-  when 'TSTP'
+  when "TSTP"
     Sidekiq.logger.info "Received TSTP, no longer accepting new work"
     launcher.quiet
-  when 'TTIN'
+  when "TTIN"
     Thread.list.each do |thread|
-      Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['label']}"
+      Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["label"]}"
       if thread.backtrace
         Sidekiq.logger.warn thread.backtrace.join("\n")
       else
@@ -89,52 +89,56 @@ def Process.rss
   `ps -o rss= -p #{Process.pid}`.chomp.to_i
 end
 
-iter = 10
+iter = 50
 count = 10_000
 
 iter.times do
-  arr = Array.new(count) do
-    []
-  end
-  count.times do |idx|
-    arr[idx][0] = idx
-  end
-  Sidekiq::Client.push_bulk('class' => LoadWorker, 'args' => arr)
+  arr = Array.new(count) { |idx| [idx] }
+  Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
 end
-Sidekiq.logger.error "Created #{count*iter} jobs"
+Sidekiq.logger.error "Created #{count * iter} jobs"
 
 start = Time.now
 
 Monitoring = Thread.new do
-  watchdog("monitor thread") do
-    while true
+  while true
+    sleep 0.2
+    qsize = Sidekiq.redis do |conn|
+      conn.llen "queue:default"
+    end
+    total = qsize
+    # Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
+    if total == 0
+      Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
+      Sidekiq.logger.error("Now here's the latency for three jobs")
+
+      LoadWorker.perform_async(1, Time.now.to_f)
+      LoadWorker.perform_async(2, Time.now.to_f)
+      LoadWorker.perform_async(3, Time.now.to_f)
+
       sleep 0.2
-      qsize = Sidekiq.redis do |conn|
-        conn.llen "queue:default"
-      end
-      total = qsize
-      #Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
-      if total == 0
-        Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
-        Sidekiq.logger.error("Now here's the latency for three jobs")
-
-        LoadWorker.perform_async(1, Time.now.to_f)
-        LoadWorker.perform_async(2, Time.now.to_f)
-        LoadWorker.perform_async(3, Time.now.to_f)
-
-        sleep 0.2
-        exit(0)
-      end
+      exit(0)
     end
   end
 end
 
+def with_latency(latency, &block)
+  Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
+  if latency > 0
+    Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
+  else
+    yield
+  end
+end
+
 begin
-  #RubyProf::exclude_threads = [ Monitoring ]
-  #RubyProf.start
-  fire_event(:startup)
-  Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
-  Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
+  # RubyProf::exclude_threads = [ Monitoring ]
+  # RubyProf.start
+  events = Sidekiq.options[:lifecycle_events][:startup]
+  events.each(&:call)
+  events.clear
+  
+  with_latency(Integer(ENV.fetch("LATENCY", "1"))) do
     launcher = Sidekiq::Launcher.new(Sidekiq.options)
     launcher.run
 
@@ -144,14 +148,14 @@ begin
     end
   end
 rescue SystemExit => e
-  #Sidekiq.logger.error("Profiling...")
-  #result = RubyProf.stop
-  #printer = RubyProf::GraphHtmlPrinter.new(result)
-  #printer.print(File.new("output.html", "w"), :min_percent => 1)
+  # Sidekiq.logger.error("Profiling...")
+  # result = RubyProf.stop
+  # printer = RubyProf::GraphHtmlPrinter.new(result)
+  # printer.print(File.new("output.html", "w"), :min_percent => 1)
   # normal
 rescue => e
   raise e if $DEBUG
-  STDERR.puts e.message
-  STDERR.puts e.backtrace.join("\n")
+  warn e.message
+  warn e.backtrace.join("\n")
   exit 1
 end

data/bin/sidekiqmon

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-require 'sidekiq/monitor'
+require "sidekiq/monitor"
 
 section = "all"
 section = ARGV[0] if ARGV.size == 1