sidekiq 6.3.1 → 6.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 33cb573c76378ff933909bdc009358cc94b81e9e50c17b6dd8160170fe744f25
-  data.tar.gz: f13c16da01f5cc0ef9514396aed899c8e1f69049d2d425892d3f5e774d68c6a7
+  metadata.gz: 99c9e264c092b88ea726be158fafe5bbab91f82f4b5864dee406280622e98e4b
+  data.tar.gz: acd72bd99929d7c9d129cb9662276cc5adb7214de07cd4fc8accf6b9d521994a
 SHA512:
-  metadata.gz: 9829304cbe0810266d2a320b14386561a4f22965aa3f9fab311253c1edd9f3075298dadba28afc0f69f898ed69d858833e744e52db1012a0d23037ecbdf33ce3
-  data.tar.gz: b38f253759ab7e3479829faad0e60e42ad2a6281b4dc022c66b0af652c53e85e5d7362bc07ecf02ae3ae3c530e3619572ad070267d2690aa5b4fafc08762d73e
+  metadata.gz: 622c25276c017302c1a9d144e9366043ba359b2c3b0c57d4e7baad8f9de2e9c9969a86c91acdbefcf736af92e297c4e1fbe2008aa41e0c1accadda77dd0724f5
+  data.tar.gz: 7e64012a5368cb0158ecaa50cdea6447709a64dd3a2816b36a31e7f17d70fffff81bd8d317c0cc1f9a6317adcffad9c200c48f9ca4bf208afba819ff7a07738e

data/Changes.md

@@ -2,6 +2,40 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
 
+HEAD
+---------
+
+- Fix pipeline/multi deprecations in redis-rb 4.6
+- Fix sidekiq.yml YAML load errors on Ruby 3.1 [#5141]
+- Sharding support for `perform_bulk` [#5129]
+- Refactor job logger for SPEEEEEEED
+
+6.4.0
+---------
+
+- **SECURITY**: Validate input to avoid possible DoS in Web UI.
+- Add **strict argument checking** [#5071]
+  Sidekiq will now log a warning if JSON-unsafe arguments are passed to `perform_async`.
+  Add `Sidekiq.strict_args!(false)` to your initializer to disable this warning.
+  This warning will switch to an exception in Sidekiq 7.0.
+- Note that Delayed Extensions will be removed in Sidekiq 7.0 [#5076]
+- Add `perform_{inline,sync}` in Sidekiq::Job to run a job synchronously [#5061, hasan-ally]
+```ruby
+SomeJob.perform_async(args...)
+SomeJob.perform_sync(args...)
+SomeJob.perform_inline(args...)
+```
+  You can also dynamically redirect a job to run synchronously:
+```ruby
+SomeJob.set("sync": true).perform_async(args...) # will run via perform_inline
+```
+- Replace Sidekiq::Worker `app/workers` generator with Sidekiq::Job `app/sidekiq` generator [#5055]
+```
+bin/rails generate sidekiq:job ProcessOrderJob
+```
+- Fix job retries losing CurrentAttributes [#5090]
+- Tweak shutdown to give long-running threads time to cleanup [#5095]
+
 6.3.1
 ---------
 

data/LICENSE

@@ -4,6 +4,6 @@ Sidekiq is an Open Source project licensed under the terms of
 the LGPLv3 license.  Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
 for license text.
 
-Sidekiq Pro has a commercial-friendly license allowing private forks
-and modifications of Sidekiq.  Please see https://sidekiq.org/products/pro.html for
-more detail.  You can find the commercial license terms in COMM-LICENSE.txt.
+Sidekiq Pro and Sidekiq Enterprise have a commercial-friendly license.
+You can find the commercial license in COMM-LICENSE.txt.
+Please see https://sidekiq.org for purchasing options.

data/README.md

@@ -43,7 +43,7 @@ Getting Started
 -----------------
 
 See the [Getting Started wiki page](https://github.com/mperham/sidekiq/wiki/Getting-Started) and follow the simple setup process.
-You can watch [this Youtube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
+You can watch [this YouTube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
 Sidekiq and see its features in action.  Here's the Web UI:
 
 ![Web UI](https://github.com/mperham/sidekiq/raw/main/examples/web-ui.png)
@@ -80,6 +80,11 @@ Useful resources:
 Every Friday morning is Sidekiq happy hour: I video chat and answer questions.
 See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
 
+Contributing
+-----------------
+
+Please see [the contributing guidelines](https://github.com/mperham/sidekiq/blob/main/.github/contributing.md).
+
 
 License
 -----------------

data/bin/sidekiq

@@ -4,7 +4,7 @@
 # RUBYOPT=-w bundle exec sidekiq
 $TESTING = false
 
-require_relative '../lib/sidekiq/cli'
+require_relative "../lib/sidekiq/cli"
 
 def integrate_with_systemd
   return unless ENV["NOTIFY_SOCKET"]
@@ -32,8 +32,8 @@ begin
 rescue => e
   raise e if $DEBUG
   if Sidekiq.error_handlers.length == 0
-    STDERR.puts e.message
-    STDERR.puts e.backtrace.join("\n")
+    warn e.message
+    warn e.backtrace.join("\n")
   else
     cli.handle_exception e
   end

data/bin/sidekiqload

@@ -4,19 +4,18 @@
 # RUBYOPT=-w bundle exec sidekiq
 $TESTING = false
 
-#require 'ruby-prof'
-require 'bundler/setup'
+# require "ruby-prof"
+require "bundler/setup"
 Bundler.require(:default, :load_test)
 
-require_relative '../lib/sidekiq/cli'
-require_relative '../lib/sidekiq/launcher'
-
-include Sidekiq::Util
+require_relative "../lib/sidekiq/cli"
+require_relative "../lib/sidekiq/launcher"
 
 Sidekiq.configure_server do |config|
   config.options[:concurrency] = 10
-  config.redis = { db: 13, port: 6380, driver: :hiredis }
-  config.options[:queues] << 'default'
+  config.redis = {db: 13, port: 6380}
+  # config.redis = { db: 13, port: 6380, driver: :hiredis}
+  config.options[:queues] << "default"
   config.logger.level = Logger::ERROR
   config.average_scheduled_poll_interval = 2
   config.reliable! if defined?(Sidekiq::Pro)
@@ -29,9 +28,9 @@ class LoadWorker
     1
   end
 
-  def perform(idx, ts=nil)
-    puts(Time.now.to_f - ts) if ts != nil
-    #raise idx.to_s if idx % 100 == 1
+  def perform(idx, ts = nil)
+    puts(Time.now.to_f - ts) if !ts.nil?
+    # raise idx.to_s if idx % 100 == 1
   end
 end
 
@@ -39,43 +38,41 @@ end
 # brew install toxiproxy
 # gem install toxiproxy
 # run `toxiproxy-server` in a separate terminal window.
-require 'toxiproxy'
+require "toxiproxy"
 # simulate a non-localhost network for realer-world conditions.
 # adding 1ms of network latency has an ENORMOUS impact on benchmarks
 Toxiproxy.populate([{
-    "name": "redis",
-    "listen": "127.0.0.1:6380",
-    "upstream": "127.0.0.1:6379"
+  name: "redis",
+  listen: "127.0.0.1:6380",
+  upstream: "127.0.0.1:6379"
 }])
 
 self_read, self_write = IO.pipe
-%w(INT TERM TSTP TTIN).each do |sig|
-  begin
-    trap sig do
-      self_write.puts(sig)
-    end
-  rescue ArgumentError
-    puts "Signal #{sig} not supported"
+%w[INT TERM TSTP TTIN].each do |sig|
+  trap sig do
+    self_write.puts(sig)
   end
+rescue ArgumentError
+  puts "Signal #{sig} not supported"
 end
 
-Sidekiq.redis {|c| c.flushdb}
+Sidekiq.redis { |c| c.flushdb }
 def handle_signal(launcher, sig)
   Sidekiq.logger.debug "Got #{sig} signal"
   case sig
-  when 'INT'
+  when "INT"
     # Handle Ctrl-C in JRuby like MRI
     # http://jira.codehaus.org/browse/JRUBY-4637
     raise Interrupt
-  when 'TERM'
+  when "TERM"
     # Heroku sends TERM and then waits 30 seconds for process to exit.
     raise Interrupt
-  when 'TSTP'
+  when "TSTP"
     Sidekiq.logger.info "Received TSTP, no longer accepting new work"
     launcher.quiet
-  when 'TTIN'
+  when "TTIN"
     Thread.list.each do |thread|
-      Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['label']}"
+      Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["label"]}"
       if thread.backtrace
         Sidekiq.logger.warn thread.backtrace.join("\n")
       else
@@ -89,7 +86,7 @@ def Process.rss
   `ps -o rss= -p #{Process.pid}`.chomp.to_i
 end
 
-iter = 10
+iter = 50
 count = 10_000
 
 iter.times do
@@ -99,40 +96,41 @@ iter.times do
   count.times do |idx|
     arr[idx][0] = idx
   end
-  Sidekiq::Client.push_bulk('class' => LoadWorker, 'args' => arr)
+  Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
 end
-Sidekiq.logger.error "Created #{count*iter} jobs"
+Sidekiq.logger.error "Created #{count * iter} jobs"
 
 start = Time.now
 
 Monitoring = Thread.new do
-  watchdog("monitor thread") do
-    while true
+  while true
+    sleep 0.2
+    qsize = Sidekiq.redis do |conn|
+      conn.llen "queue:default"
+    end
+    total = qsize
+    # Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
+    if total == 0
+      Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
+      Sidekiq.logger.error("Now here's the latency for three jobs")
+
+      LoadWorker.perform_async(1, Time.now.to_f)
+      LoadWorker.perform_async(2, Time.now.to_f)
+      LoadWorker.perform_async(3, Time.now.to_f)
+
       sleep 0.2
-      qsize = Sidekiq.redis do |conn|
-        conn.llen "queue:default"
-      end
-      total = qsize
-      #Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
-      if total == 0
-        Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
-        Sidekiq.logger.error("Now here's the latency for three jobs")
-
-        LoadWorker.perform_async(1, Time.now.to_f)
-        LoadWorker.perform_async(2, Time.now.to_f)
-        LoadWorker.perform_async(3, Time.now.to_f)
-
-        sleep 0.2
-        exit(0)
-      end
+      exit(0)
     end
   end
 end
 
 begin
-  #RubyProf::exclude_threads = [ Monitoring ]
-  #RubyProf.start
-  fire_event(:startup)
+  # RubyProf::exclude_threads = [ Monitoring ]
+  # RubyProf.start
+  events = Sidekiq.options[:lifecycle_events][:startup]
+  events.each(&:call)
+  events.clear
+
   Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
   Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
     launcher = Sidekiq::Launcher.new(Sidekiq.options)
@@ -144,14 +142,14 @@ begin
     end
   end
 rescue SystemExit => e
-  #Sidekiq.logger.error("Profiling...")
-  #result = RubyProf.stop
-  #printer = RubyProf::GraphHtmlPrinter.new(result)
-  #printer.print(File.new("output.html", "w"), :min_percent => 1)
+  # Sidekiq.logger.error("Profiling...")
+  # result = RubyProf.stop
+  # printer = RubyProf::GraphHtmlPrinter.new(result)
+  # printer.print(File.new("output.html", "w"), :min_percent => 1)
   # normal
 rescue => e
   raise e if $DEBUG
-  STDERR.puts e.message
-  STDERR.puts e.backtrace.join("\n")
+  warn e.message
+  warn e.backtrace.join("\n")
   exit 1
 end

data/bin/sidekiqmon

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-require 'sidekiq/monitor'
+require "sidekiq/monitor"
 
 section = "all"
 section = ARGV[0] if ARGV.size == 1

data/lib/generators/sidekiq/job_generator.rb

@@ -0,0 +1,57 @@
+require "rails/generators/named_base"
+
+module Sidekiq
+  module Generators # :nodoc:
+    class JobGenerator < ::Rails::Generators::NamedBase # :nodoc:
+      desc "This generator creates a Sidekiq Job in app/sidekiq and a corresponding test"
+
+      check_class_collision suffix: "Job"
+
+      def self.default_generator_root
+        File.dirname(__FILE__)
+      end
+
+      def create_job_file
+        template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
+      end
+
+      def create_test_file
+        return unless test_framework
+
+        if test_framework == :rspec
+          create_job_spec
+        else
+          create_job_test
+        end
+      end
+
+      private
+
+      def create_job_spec
+        template_file = File.join(
+          "spec/sidekiq",
+          class_path,
+          "#{file_name}_job_spec.rb"
+        )
+        template "job_spec.rb.erb", template_file
+      end
+
+      def create_job_test
+        template_file = File.join(
+          "test/sidekiq",
+          class_path,
+          "#{file_name}_job_test.rb"
+        )
+        template "job_test.rb.erb", template_file
+      end
+
+      def file_name
+        @_file_name ||= super.sub(/_?job\z/i, "")
+      end
+
+      def test_framework
+        ::Rails.application.config.generators.options[:rails][:test_framework]
+      end
+    end
+  end
+end

data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb}

@@ -1,6 +1,6 @@
 <% module_namespacing do -%>
-class <%= class_name %>Worker
-  include Sidekiq::Worker
+class <%= class_name %>Job
+  include Sidekiq::Job
 
   def perform(*args)
     # Do something

data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb}

@@ -1,6 +1,6 @@
 require 'rails_helper'
 <% module_namespacing do -%>
-RSpec.describe <%= class_name %>Worker, type: :worker do
+RSpec.describe <%= class_name %>Job, type: :job do
   pending "add some examples to (or delete) #{__FILE__}"
 end
 <% end -%>

data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb}

@@ -1,6 +1,6 @@
 require 'test_helper'
 <% module_namespacing do -%>
-class <%= class_name %>WorkerTest < Minitest::Test
+class <%= class_name %>JobTest < Minitest::Test
   def test_example
     skip "add some examples to (or delete) #{__FILE__}"
   end

data/lib/sidekiq/api.rb

@@ -54,14 +54,14 @@ module Sidekiq
     # O(1) redis calls
     def fetch_stats_fast!
       pipe1_res = Sidekiq.redis { |conn|
-        conn.pipelined do
-          conn.get("stat:processed")
-          conn.get("stat:failed")
-          conn.zcard("schedule")
-          conn.zcard("retry")
-          conn.zcard("dead")
-          conn.scard("processes")
-          conn.lrange("queue:default", -1, -1)
+        conn.pipelined do |pipeline|
+          pipeline.get("stat:processed")
+          pipeline.get("stat:failed")
+          pipeline.zcard("schedule")
+          pipeline.zcard("retry")
+          pipeline.zcard("dead")
+          pipeline.scard("processes")
+          pipeline.lrange("queue:default", -1, -1)
         end
       }
 
@@ -101,9 +101,9 @@ module Sidekiq
       }
 
       pipe2_res = Sidekiq.redis { |conn|
-        conn.pipelined do
-          processes.each { |key| conn.hget(key, "busy") }
-          queues.each { |queue| conn.llen("queue:#{queue}") }
+        conn.pipelined do |pipeline|
+          processes.each { |key| pipeline.hget(key, "busy") }
+          queues.each { |queue| pipeline.llen("queue:#{queue}") }
         end
       }
 
@@ -147,9 +147,9 @@ module Sidekiq
         Sidekiq.redis do |conn|
           queues = conn.sscan_each("queues").to_a
 
-          lengths = conn.pipelined {
+          lengths = conn.pipelined { |pipeline|
             queues.each do |queue|
-              conn.llen("queue:#{queue}")
+              pipeline.llen("queue:#{queue}")
             end
           }
 
@@ -161,6 +161,8 @@ module Sidekiq
 
     class History
       def initialize(days_previous, start_date = nil)
+        # we only store five years of data in Redis
+        raise ArgumentError if days_previous < 1 || days_previous > (5 * 365)
         @days_previous = days_previous
         @start_date = start_date || Time.now.utc.to_date
       end
@@ -285,9 +287,9 @@ module Sidekiq
 
     def clear
       Sidekiq.redis do |conn|
-        conn.multi do
-          conn.unlink(@rname)
-          conn.srem("queues", name)
+        conn.multi do |transaction|
+          transaction.unlink(@rname)
+          transaction.srem("queues", name)
         end
       end
     end
@@ -353,8 +355,12 @@ module Sidekiq
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
       @display_args ||= case klass
                 when /\ASidekiq::Extensions::Delayed/
-                  safe_load(args[0], args) do |_, _, arg|
-                    arg
+                  safe_load(args[0], args) do |_, _, arg, kwarg|
+                    if !kwarg || kwarg.empty?
+                      arg
+                    else
+                      [arg, kwarg]
+                    end
                   end
                 when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
                   job_args = self["wrapped"] ? args[0]["arguments"] : []
@@ -517,9 +523,9 @@ module Sidekiq
 
     def remove_job
       Sidekiq.redis do |conn|
-        results = conn.multi {
-          conn.zrangebyscore(parent.name, score, score)
-          conn.zremrangebyscore(parent.name, score, score)
+        results = conn.multi { |transaction|
+          transaction.zrangebyscore(parent.name, score, score)
+          transaction.zremrangebyscore(parent.name, score, score)
         }.first
 
         if results.size == 1
@@ -540,9 +546,9 @@ module Sidekiq
           yield msg if msg
 
           # push the rest back onto the sorted set
-          conn.multi do
+          conn.multi do |transaction|
             nonmatched.each do |message|
-              conn.zadd(parent.name, score.to_f.to_s, message)
+              transaction.zadd(parent.name, score.to_f.to_s, message)
             end
           end
         end
@@ -729,10 +735,10 @@ module Sidekiq
     def kill(message, opts = {})
       now = Time.now.to_f
       Sidekiq.redis do |conn|
-        conn.multi do
-          conn.zadd(name, now.to_s, message)
-          conn.zremrangebyscore(name, "-inf", now - self.class.timeout)
-          conn.zremrangebyrank(name, 0, - self.class.max_jobs)
+        conn.multi do |transaction|
+          transaction.zadd(name, now.to_s, message)
+          transaction.zremrangebyscore(name, "-inf", now - self.class.timeout)
+          transaction.zremrangebyrank(name, 0, - self.class.max_jobs)
         end
       end
 
@@ -780,9 +786,9 @@ module Sidekiq
       count = 0
       Sidekiq.redis do |conn|
         procs = conn.sscan_each("processes").to_a.sort
-        heartbeats = conn.pipelined {
+        heartbeats = conn.pipelined { |pipeline|
           procs.each do |key|
-            conn.hget(key, "info")
+            pipeline.hget(key, "info")
           end
         }
 
@@ -804,9 +810,9 @@ module Sidekiq
         # We're making a tradeoff here between consuming more memory instead of
         # making more roundtrips to Redis, but if you have hundreds or thousands of workers,
         # you'll be happier this way
-        conn.pipelined do
+        conn.pipelined do |pipeline|
           procs.each do |key|
-            conn.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
+            pipeline.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
           end
         end
       }
@@ -819,10 +825,10 @@ module Sidekiq
 
         hash = Sidekiq.load_json(info)
         yield Process.new(hash.merge("busy" => busy.to_i,
-                                     "beat" => at_s.to_f,
-                                     "quiet" => quiet,
-                                     "rss" => rss.to_i,
-                                     "rtt_us" => rtt.to_i))
+          "beat" => at_s.to_f,
+          "quiet" => quiet,
+          "rss" => rss.to_i,
+          "rtt_us" => rtt.to_i))
       end
     end
 
@@ -920,9 +926,9 @@ module Sidekiq
     def signal(sig)
       key = "#{identity}-signals"
       Sidekiq.redis do |c|
-        c.multi do
-          c.lpush(key, sig)
-          c.expire(key, 60)
+        c.multi do |transaction|
+          transaction.lpush(key, sig)
+          transaction.expire(key, 60)
         end
       end
     end
@@ -956,9 +962,9 @@ module Sidekiq
       Sidekiq.redis do |conn|
         procs = conn.sscan_each("processes").to_a
         procs.sort.each do |key|
-          valid, workers = conn.pipelined {
-            conn.exists?(key)
-            conn.hgetall("#{key}:workers")
+          valid, workers = conn.pipelined { |pipeline|
+            pipeline.exists?(key)
+            pipeline.hgetall("#{key}:workers")
           }
           next unless valid
           workers.each_pair do |tid, json|
@@ -986,9 +992,9 @@ module Sidekiq
         if procs.empty?
           0
         else
-          conn.pipelined {
+          conn.pipelined { |pipeline|
             procs.each do |key|
-              conn.hget(key, "busy")
+              pipeline.hget(key, "busy")
             end
           }.sum(&:to_i)
         end

data/lib/sidekiq/cli.rb

@@ -115,8 +115,8 @@ module Sidekiq
       begin
         launcher.run
 
-        while (readable_io = IO.select([self_read]))
-          signal = readable_io.first[0].gets.strip
+        while (readable_io = self_read.wait_readable)
+          signal = readable_io.gets.strip
           handle_signal(signal)
         end
       rescue Interrupt
@@ -380,7 +380,9 @@ module Sidekiq
     end
 
     def parse_config(path)
-      opts = YAML.load(ERB.new(File.read(path)).result) || {}
+      erb = ERB.new(File.read(path))
+      erb.filename = File.expand_path(path)
+      opts = load_yaml(erb.result) || {}
 
       if opts.respond_to? :deep_symbolize_keys!
         opts.deep_symbolize_keys!
@@ -396,6 +398,14 @@ module Sidekiq
       opts
     end
 
+    def load_yaml(src)
+      if Psych::VERSION > "4.0"
+        YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
+      else
+        YAML.load(src)
+      end
+    end
+
     def parse_queues(opts, queues_and_weights)
       queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
     end