sidekiq 6.4.1 → 6.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 99c9e264c092b88ea726be158fafe5bbab91f82f4b5864dee406280622e98e4b
-  data.tar.gz: acd72bd99929d7c9d129cb9662276cc5adb7214de07cd4fc8accf6b9d521994a
+  metadata.gz: 1afbc6a1a0b14403e9148e746c08e0a2b24e634fca05288982c96719675607de
+  data.tar.gz: 3ff3f8df76b565f42030462eb8d09b673b89751dcd6a0b7c41a255789960d321
 SHA512:
-  metadata.gz: 622c25276c017302c1a9d144e9366043ba359b2c3b0c57d4e7baad8f9de2e9c9969a86c91acdbefcf736af92e297c4e1fbe2008aa41e0c1accadda77dd0724f5
-  data.tar.gz: 7e64012a5368cb0158ecaa50cdea6447709a64dd3a2816b36a31e7f17d70fffff81bd8d317c0cc1f9a6317adcffad9c200c48f9ca4bf208afba819ff7a07738e
+  metadata.gz: e8a68611735322d98cc517f1d03ef02394497f8eb505e0db496909cac1f6b7f0179f38ce1966546f717115395102c78fdad9ae0fd947360307b9288dcc22b369
+  data.tar.gz: 163e41dfb153a4e2ec50d407bde08cb8a395909807053f24478a73af1a6a9858b5aec92cbf618d4d066560a2b7bedaa2f88b48f823609e2d78f21320270cb97e

data/Changes.md

@@ -2,7 +2,46 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
 
-HEAD
+6.5.1
+----------
+
+- Fix `push_bulk` breakage [#5387]
+
+6.5.0
+---------
+
+- Substantial refactoring of Sidekiq server internals, part of a larger effort
+  to reduce Sidekiq's internal usage of global methods and data, see [docs/global_to_local.md](docs/global_to_local.md) and [docs/middleware.md](docs/middleware.md).
+- **Add beta support for the `redis-client` gem**. This will become the default Redis driver in Sidekiq 7.0. [#5298]
+  Read more: https://github.com/mperham/sidekiq/wiki/Using-redis-client
+- **Add beta support for DB transaction-aware client** [#5291]
+  Add this line to your initializer and any jobs created during a transaction
+  will only be pushed to Redis **after the transaction commits**. You will need to add the
+  `after_commit_everywhere` gem to your Gemfile.
+```ruby
+Sidekiq.transactional_push!
+```
+  This feature does not have a lot of production usage yet; please try it out and let us
+  know if you have any issues. It will be fully supported in Sidekiq 7.0 or removed if it
+  proves problematic.
+- Fix regression with middleware arguments [#5312]
+
+6.4.2
+---------
+
+- Strict argument checking now runs after client-side middleware [#5246]
+- Fix page events with live polling [#5184]
+- Many under-the-hood changes to remove all usage of the term "worker"
+  from the Sidekiq codebase and APIs. This mostly involved RDoc and local
+  variable names but a few constants and public APIs were changed. The old
+  APIs will be removed in Sidekiq 7.0.
+```
+Sidekiq::DEFAULT_WORKER_OPTIONS -> Sidekiq.default_job_options
+Sidekiq.default_worker_options -> Sidekiq.default_job_options
+Sidekiq::Queues["default"].jobs_by_worker(HardJob) -> Sidekiq::Queues["default"].jobs_by_class(HardJob)
+```
+
+6.4.1
 ---------
 
 - Fix pipeline/multi deprecations in redis-rb 4.6
@@ -319,6 +358,13 @@ See the [Logging wiki page](https://github.com/mperham/sidekiq/wiki/Logging) for
 - Integrate the StandardRB code formatter to ensure consistent code
   styling. [#4114, gearnode]
 
+5.2.10
+---------
+
+- Backport fix for CVE-2022-23837.
+- Migrate to `exists?` for redis-rb.
+- Lock redis-rb to <4.6 to avoid deprecations.
+
 5.2.9
 ---------
 

data/README.md

@@ -36,7 +36,7 @@ Sidekiq 6.0 supports Rails 5.0+ but does not require it.
 Installation
 -----------------
 
-    gem install sidekiq
+    bundle add sidekiq
 
 
 Getting Started

data/bin/sidekiqload

@@ -11,6 +11,10 @@ Bundler.require(:default, :load_test)
 require_relative "../lib/sidekiq/cli"
 require_relative "../lib/sidekiq/launcher"
 
+if ENV["SIDEKIQ_REDIS_CLIENT"]
+  Sidekiq::RedisConnection.adapter = :redis_client
+end
+
 Sidekiq.configure_server do |config|
   config.options[:concurrency] = 10
   config.redis = {db: 13, port: 6380}
@@ -36,7 +40,6 @@ end
 
 # brew tap shopify/shopify
 # brew install toxiproxy
-# gem install toxiproxy
 # run `toxiproxy-server` in a separate terminal window.
 require "toxiproxy"
 # simulate a non-localhost network for realer-world conditions.
@@ -90,12 +93,7 @@ iter = 50
 count = 10_000
 
 iter.times do
-  arr = Array.new(count) do
-    []
-  end
-  count.times do |idx|
-    arr[idx][0] = idx
-  end
+  arr = Array.new(count) { |idx| [idx] }
   Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
 end
 Sidekiq.logger.error "Created #{count * iter} jobs"
@@ -124,15 +122,23 @@ Monitoring = Thread.new do
   end
 end
 
+def with_latency(latency, &block)
+  Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
+  if latency > 0
+    Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
+  else
+    yield
+  end
+end
+
 begin
   # RubyProf::exclude_threads = [ Monitoring ]
   # RubyProf.start
   events = Sidekiq.options[:lifecycle_events][:startup]
   events.each(&:call)
   events.clear
-
-  Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
-  Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
+  
+  with_latency(Integer(ENV.fetch("LATENCY", "1"))) do
     launcher = Sidekiq::Launcher.new(Sidekiq.options)
     launcher.run
 

data/lib/sidekiq/api.rb

@@ -191,7 +191,7 @@ module Sidekiq
               stat_hash[dates[idx]] = value ? value.to_i : 0
             end
           end
-        rescue Redis::CommandError
+        rescue RedisConnection.adapter::CommandError
           # mget will trigger a CROSSSLOT error when run against a Cluster
           # TODO Someone want to add Cluster support?
         end
@@ -217,24 +217,30 @@ module Sidekiq
     include Enumerable
 
     ##
-    # Return all known queues within Redis.
+    # Fetch all known queues within Redis.
     #
+    # @return [Array<Sidekiq::Queue>]
     def self.all
       Sidekiq.redis { |c| c.sscan_each("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
     end
 
     attr_reader :name
 
+    # @param name [String] the name of the queue
     def initialize(name = "default")
       @name = name.to_s
       @rname = "queue:#{name}"
     end
 
+    # The current size of the queue within Redis.
+    # This value is real-time and can change between calls.
+    #
+    # @return [Integer] the size
     def size
       Sidekiq.redis { |con| con.llen(@rname) }
     end
 
-    # Sidekiq Pro overrides this
+    # @return [Boolean] if the queue is currently paused
     def paused?
       false
     end
@@ -243,7 +249,7 @@ module Sidekiq
     # Calculates this queue's latency, the difference in seconds since the oldest
     # job in the queue was enqueued.
     #
-    # @return Float
+    # @return [Float] in seconds
     def latency
       entry = Sidekiq.redis { |conn|
         conn.lrange(@rname, -1, -1)
@@ -279,12 +285,17 @@ module Sidekiq
     ##
     # Find the job with the given JID within this queue.
     #
-    # This is a slow, inefficient operation.  Do not use under
+    # This is a *slow, inefficient* operation.  Do not use under
     # normal conditions.
+    #
+    # @param jid [String] the job_id to look for
+    # @return [Sidekiq::JobRecord]
+    # @return [nil] if not found
     def find_job(jid)
       detect { |j| j.jid == jid }
     end
 
+    # delete all jobs within this queue
     def clear
       Sidekiq.redis do |conn|
         conn.multi do |transaction|
@@ -294,6 +305,10 @@ module Sidekiq
       end
     end
     alias_method :💣, :clear
+
+    def as_json(options = nil) # :nodoc:
+      {name: name} # 5336
+    end
   end
 
   ##
@@ -306,15 +321,16 @@ module Sidekiq
   class JobRecord
     attr_reader :item
     attr_reader :value
+    attr_reader :queue
 
-    def initialize(item, queue_name = nil)
+    def initialize(item, queue_name = nil) # :nodoc:
       @args = nil
       @value = item
       @item = item.is_a?(Hash) ? item : parse(item)
       @queue = queue_name || @item["queue"]
     end
 
-    def parse(item)
+    def parse(item) # :nodoc:
       Sidekiq.load_json(item)
     rescue JSON::ParserError
       # If the job payload in Redis is invalid JSON, we'll load
@@ -354,31 +370,31 @@ module Sidekiq
     def display_args
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
       @display_args ||= case klass
-                when /\ASidekiq::Extensions::Delayed/
-                  safe_load(args[0], args) do |_, _, arg, kwarg|
-                    if !kwarg || kwarg.empty?
-                      arg
-                    else
-                      [arg, kwarg]
-                    end
-                  end
-                when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
-                  job_args = self["wrapped"] ? args[0]["arguments"] : []
-                  if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
-                    # remove MailerClass, mailer_method and 'deliver_now'
-                    job_args.drop(3)
-                  elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
-                    # remove MailerClass, mailer_method and 'deliver_now'
-                    job_args.drop(3).first["args"]
-                  else
-                    job_args
-                  end
-                else
-                  if self["encrypt"]
-                    # no point in showing 150+ bytes of random garbage
-                    args[-1] = "[encrypted data]"
-                  end
-                  args
+      when /\ASidekiq::Extensions::Delayed/
+        safe_load(args[0], args) do |_, _, arg, kwarg|
+          if !kwarg || kwarg.empty?
+            arg
+          else
+            [arg, kwarg]
+          end
+        end
+      when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+        job_args = self["wrapped"] ? args[0]["arguments"] : []
+        if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
+          # remove MailerClass, mailer_method and 'deliver_now'
+          job_args.drop(3)
+        elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
+          # remove MailerClass, mailer_method and 'deliver_now'
+          job_args.drop(3).first["args"]
+        else
+          job_args
+        end
+      else
+        if self["encrypt"]
+          # no point in showing 150+ bytes of random garbage
+          args[-1] = "[encrypted data]"
+        end
+        args
       end
     end
 
@@ -412,15 +428,12 @@ module Sidekiq
       end
     end
 
-    attr_reader :queue
-
     def latency
       now = Time.now.to_f
       now - (@item["enqueued_at"] || @item["created_at"] || now)
     end
 
-    ##
-    # Remove this job from the queue.
+    # Remove this job from the queue
     def delete
       count = Sidekiq.redis { |conn|
         conn.lrem("queue:#{@queue}", 1, @value)
@@ -428,6 +441,7 @@ module Sidekiq
       count != 0
     end
 
+    # Access arbitrary attributes within the job hash
     def [](name)
       # nil will happen if the JSON fails to parse.
       # We don't guarantee Sidekiq will work with bad job JSON but we should
@@ -442,7 +456,8 @@ module Sidekiq
     rescue => ex
       # #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
       # memory yet so the YAML can't be loaded.
-      Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
+      # TODO is this still necessary? Zeitwerk reloader should handle?
+      Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.config[:environment] == "development"
       default
     end
 
@@ -464,13 +479,15 @@ module Sidekiq
     end
   end
 
+  # Represents a job within a Redis sorted set where the score
+  # represents a timestamp for the job.
   class SortedEntry < JobRecord
     attr_reader :score
     attr_reader :parent
 
-    def initialize(parent, score, item)
+    def initialize(parent, score, item) # :nodoc:
       super(item)
-      @score = score
+      @score = Float(score)
       @parent = parent
     end
 
@@ -486,12 +503,17 @@ module Sidekiq
       end
     end
 
+    # Change the scheduled time for this job.
+    #
+    # @param [Time] the new timestamp when this job will be enqueued.
     def reschedule(at)
       Sidekiq.redis do |conn|
         conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
       end
     end
 
+    # Enqueue this job from the scheduled or dead set so it will
+    # be executed at some point in the near future.
     def add_to_queue
       remove_job do |message|
         msg = Sidekiq.load_json(message)
@@ -499,6 +521,8 @@ module Sidekiq
       end
     end
 
+    # enqueue this job from the retry set so it will be executed
+    # at some point in the near future.
     def retry
       remove_job do |message|
         msg = Sidekiq.load_json(message)
@@ -507,8 +531,7 @@ module Sidekiq
       end
     end
 
-    ##
-    # Place job in the dead set
+    # Move this job from its current set into the Dead set.
     def kill
       remove_job do |message|
         DeadSet.new.kill(message)
@@ -587,6 +610,10 @@ module Sidekiq
       end
     end
     alias_method :💣, :clear
+
+    def as_json(options = nil) # :nodoc:
+      {name: name} # 5336
+    end
   end
 
   class JobSet < SortedSet
@@ -606,7 +633,7 @@ module Sidekiq
         range_start = page * page_size + offset_size
         range_end = range_start + page_size - 1
         elements = Sidekiq.redis { |conn|
-          conn.zrange name, range_start, range_end, with_scores: true
+          conn.zrange name, range_start, range_end, withscores: true
         }
         break if elements.empty?
         page -= 1
@@ -629,7 +656,7 @@ module Sidekiq
         end
 
       elements = Sidekiq.redis { |conn|
-        conn.zrangebyscore(name, begin_score, end_score, with_scores: true)
+        conn.zrangebyscore(name, begin_score, end_score, withscores: true)
       }
 
       elements.each_with_object([]) do |element, result|
@@ -758,11 +785,11 @@ module Sidekiq
     end
 
     def self.max_jobs
-      Sidekiq.options[:dead_max_jobs]
+      Sidekiq[:dead_max_jobs]
     end
 
     def self.timeout
-      Sidekiq.options[:dead_timeout_in_seconds]
+      Sidekiq[:dead_timeout_in_seconds]
     end
   end
 
@@ -964,7 +991,7 @@ module Sidekiq
         procs.sort.each do |key|
           valid, workers = conn.pipelined { |pipeline|
             pipeline.exists?(key)
-            pipeline.hgetall("#{key}:workers")
+            pipeline.hgetall("#{key}:work")
           }
           next unless valid
           workers.each_pair do |tid, json|

data/lib/sidekiq/cli.rb

@@ -9,18 +9,23 @@ require "erb"
 require "fileutils"
 
 require "sidekiq"
+require "sidekiq/component"
 require "sidekiq/launcher"
-require "sidekiq/util"
 
-module Sidekiq
+module Sidekiq # :nodoc:
   class CLI
-    include Util
+    include Sidekiq::Component
     include Singleton unless $TESTING
 
     attr_accessor :launcher
     attr_accessor :environment
+    attr_accessor :config
+
+    def parse(args = ARGV.dup)
+      @config = Sidekiq
+      @config[:error_handlers].clear
+      @config[:error_handlers] << @config.method(:default_error_handler)
 
-    def parse(args = ARGV)
       setup_options(args)
       initialize_logger
       validate!
@@ -36,7 +41,7 @@ module Sidekiq
     def run(boot_app: true)
       boot_application if boot_app
 
-      if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
+      if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
         print_banner
       end
       logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
@@ -67,7 +72,7 @@ module Sidekiq
 
       # touch the connection pool so it is created before we
       # fire startup and start multithreading.
-      info = Sidekiq.redis_info
+      info = @config.redis_info
       ver = info["redis_version"]
       raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
 
@@ -85,22 +90,22 @@ module Sidekiq
 
       # Since the user can pass us a connection pool explicitly in the initializer, we
       # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
-      cursize = Sidekiq.redis_pool.size
-      needed = Sidekiq.options[:concurrency] + 2
+      cursize = @config.redis_pool.size
+      needed = @config[:concurrency] + 2
       raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
 
       # cache process identity
-      Sidekiq.options[:identity] = identity
+      @config[:identity] = identity
 
       # Touch middleware so it isn't lazy loaded by multiple threads, #3043
-      Sidekiq.server_middleware
+      @config.server_middleware
 
       # Before this point, the process is initializing with just the main thread.
       # Starting here the process will now have multiple threads running.
       fire_event(:startup, reverse: false, reraise: true)
 
-      logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(", ")}" }
-      logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(", ")}" }
+      logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
+      logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
 
       launch(self_read)
     end
@@ -110,13 +115,13 @@ module Sidekiq
         logger.info "Starting processing, hit Ctrl-C to stop"
       end
 
-      @launcher = Sidekiq::Launcher.new(options)
+      @launcher = Sidekiq::Launcher.new(@config)
 
       begin
         launcher.run
 
-        while (readable_io = self_read.wait_readable)
-          signal = readable_io.gets.strip
+        while self_read.wait_readable
+          signal = self_read.gets.strip
           handle_signal(signal)
         end
       rescue Interrupt
@@ -173,25 +178,25 @@ module Sidekiq
       # Heroku sends TERM and then waits 30 seconds for process to exit.
       "TERM" => ->(cli) { raise Interrupt },
       "TSTP" => ->(cli) {
-        Sidekiq.logger.info "Received TSTP, no longer accepting new work"
+        cli.logger.info "Received TSTP, no longer accepting new work"
         cli.launcher.quiet
       },
       "TTIN" => ->(cli) {
         Thread.list.each do |thread|
-          Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
+          cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
           if thread.backtrace
-            Sidekiq.logger.warn thread.backtrace.join("\n")
+            cli.logger.warn thread.backtrace.join("\n")
           else
-            Sidekiq.logger.warn "<no backtrace available>"
+            cli.logger.warn "<no backtrace available>"
           end
         end
       }
     }
-    UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
+    UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
     SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
 
     def handle_signal(sig)
-      Sidekiq.logger.debug "Got #{sig} signal"
+      logger.debug "Got #{sig} signal"
       SIGNAL_HANDLERS[sig].call(self)
     end
 
@@ -237,7 +242,7 @@ module Sidekiq
         config_dir = if File.directory?(opts[:require].to_s)
           File.join(opts[:require], "config")
         else
-          File.join(options[:require], "config")
+          File.join(@config[:require], "config")
         end
 
         %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
@@ -254,27 +259,23 @@ module Sidekiq
       opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
 
       # merge with defaults
-      options.merge!(opts)
-    end
-
-    def options
-      Sidekiq.options
+      @config.merge!(opts)
     end
 
     def boot_application
       ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
 
-      if File.directory?(options[:require])
+      if File.directory?(@config[:require])
         require "rails"
         if ::Rails::VERSION::MAJOR < 5
           raise "Sidekiq no longer supports this version of Rails"
         else
           require "sidekiq/rails"
-          require File.expand_path("#{options[:require]}/config/environment.rb")
+          require File.expand_path("#{@config[:require]}/config/environment.rb")
         end
-        options[:tag] ||= default_tag
+        @config[:tag] ||= default_tag
       else
-        require options[:require]
+        require @config[:require]
       end
     end
 
@@ -291,18 +292,18 @@ module Sidekiq
     end
 
     def validate!
-      if !File.exist?(options[:require]) ||
-          (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
+      if !File.exist?(@config[:require]) ||
+          (File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
         logger.info "=================================================================="
         logger.info "  Please point Sidekiq to a Rails application or a Ruby file  "
-        logger.info "  to load your worker classes with -r [DIR|FILE]."
+        logger.info "  to load your job classes with -r [DIR|FILE]."
         logger.info "=================================================================="
         logger.info @parser
         die(1)
       end
 
       [:concurrency, :timeout].each do |opt|
-        raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.key?(opt) && options[opt].to_i <= 0
+        raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
       end
     end
 
@@ -336,7 +337,7 @@ module Sidekiq
           parse_queue opts, queue, weight
         end
 
-        o.on "-r", "--require [PATH|DIR]", "Location of Rails application with workers or file to require" do |arg|
+        o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
           opts[:require] = arg
         end
 
@@ -376,7 +377,7 @@ module Sidekiq
     end
 
     def initialize_logger
-      Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
+      @config.logger.level = ::Logger::DEBUG if @config[:verbose]
     end
 
     def parse_config(path)