sidekiq 6.4.2 → 6.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70d899b76bdd764a2ec7e5f23b6d056494b58897daaf49db672daad3e0f60237
-  data.tar.gz: daae0430ecaf8eb7f172c70e628581c20a17e1a7cb1a58170b9082fc3dde1ea2
+  metadata.gz: dedc78179d612e64435e57f0f6644c2645c06ffdc24151c9ad51df4035b51efa
+  data.tar.gz: 5987eba09190793cfccbff56066b54b5308a1fe74be7551cb5b9d6d03d256c3a
 SHA512:
-  metadata.gz: 930a0feb7ff47473eb995f8ba695468498afecb311c25296c93042ecfa00cd2bf48e19efd5ba775b4895c065705a344043e0c0e5d7b4115c2dceb1ec7cf341f9
-  data.tar.gz: 14ab645609e7c9f3fa954432097cd46d6e9e74e5bddc9ed1541697d13f3399add407ffb5af23c89c82651793c97e2c54b8fd4f845ca20287ce6bf033fd9caed2
+  metadata.gz: 9a26b5eb6e2248870f07ede054cac8330b52ad548d27b40d3950b6d985ef5ef8359e4dfc062757dc4b479036f0b2716bacfc821a20bd63413ea7c3f772bc6ead
+  data.tar.gz: 46a35f7f27693364b1b6077e602d3f69e476508654157f48fcd457a70bbcdb563f8515812ef37abae207604596e1d42f78132ac54471109f913e87c11a707cdb

data/Changes.md

@@ -2,6 +2,26 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
 
+6.5.0
+---------
+
+- Substantial refactoring of Sidekiq server internals, part of a larger effort
+  to reduce Sidekiq's internal usage of global methods and data, see [docs/component.md](docs/component.md),
+  [docs/global_to_local.md](docs/global_to_local.md) and [docs/middleware.md](docs/middleware.md).
+- **Add beta support for the `redis-client` gem**. This will become the default Redis driver in Sidekiq 7.0. [#5298]
+  Read more: https://github.com/mperham/sidekiq/wiki/Using-redis-client
+- **Add beta support for DB transaction-aware client** [#5291]
+  Add this line to your initializer and any jobs created during a transaction
+  will only be pushed to Redis **after the transaction commits**. You will need to add the
+  `after_commit_everywhere` gem to your Gemfile.
+```ruby
+Sidekiq.transactional_push!
+```
+  This feature does not have a lot of production usage yet; please try it out and let us
+  know if you have any issues. It will be fully supported in Sidekiq 7.0 or removed if it
+  proves problematic.
+- Fix regression with middleware arguments [#5312]
+
 6.4.2
 ---------
 

data/bin/sidekiqload

@@ -11,6 +11,10 @@ Bundler.require(:default, :load_test)
 require_relative "../lib/sidekiq/cli"
 require_relative "../lib/sidekiq/launcher"
 
+if ENV["SIDEKIQ_REDIS_CLIENT"]
+  Sidekiq::RedisConnection.adapter = :redis_client
+end
+
 Sidekiq.configure_server do |config|
   config.options[:concurrency] = 10
   config.redis = {db: 13, port: 6380}
@@ -118,15 +122,23 @@ Monitoring = Thread.new do
   end
 end
 
+def with_latency(latency, &block)
+  Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
+  if latency > 0
+    Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
+  else
+    yield
+  end
+end
+
 begin
   # RubyProf::exclude_threads = [ Monitoring ]
   # RubyProf.start
   events = Sidekiq.options[:lifecycle_events][:startup]
   events.each(&:call)
   events.clear
-
-  Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
-  Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
+  
+  with_latency(Integer(ENV.fetch("LATENCY", "1"))) do
     launcher = Sidekiq::Launcher.new(Sidekiq.options)
     launcher.run
 

data/lib/sidekiq/api.rb

@@ -191,7 +191,7 @@ module Sidekiq
               stat_hash[dates[idx]] = value ? value.to_i : 0
             end
           end
-        rescue Redis::CommandError
+        rescue RedisConnection.adapter::CommandError
           # mget will trigger a CROSSSLOT error when run against a Cluster
           # TODO Someone want to add Cluster support?
         end
@@ -294,6 +294,10 @@ module Sidekiq
       end
     end
     alias_method :💣, :clear
+
+    def as_json(options = nil) # :nodoc:
+      {name: name} # 5336
+    end
   end
 
   ##
@@ -442,7 +446,8 @@ module Sidekiq
     rescue => ex
       # #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
       # memory yet so the YAML can't be loaded.
-      Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
+      # TODO is this still necessary? Zeitwerk reloader should handle?
+      Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.config[:environment] == "development"
       default
     end
 
@@ -470,7 +475,7 @@ module Sidekiq
 
     def initialize(parent, score, item)
       super(item)
-      @score = score
+      @score = Float(score)
       @parent = parent
     end
 
@@ -587,6 +592,10 @@ module Sidekiq
       end
     end
     alias_method :💣, :clear
+
+    def as_json(options = nil) # :nodoc:
+      {name: name} # 5336
+    end
   end
 
   class JobSet < SortedSet
@@ -606,7 +615,7 @@ module Sidekiq
         range_start = page * page_size + offset_size
         range_end = range_start + page_size - 1
         elements = Sidekiq.redis { |conn|
-          conn.zrange name, range_start, range_end, with_scores: true
+          conn.zrange name, range_start, range_end, withscores: true
         }
         break if elements.empty?
         page -= 1
@@ -629,7 +638,7 @@ module Sidekiq
         end
 
       elements = Sidekiq.redis { |conn|
-        conn.zrangebyscore(name, begin_score, end_score, with_scores: true)
+        conn.zrangebyscore(name, begin_score, end_score, withscores: true)
       }
 
       elements.each_with_object([]) do |element, result|
@@ -758,11 +767,11 @@ module Sidekiq
     end
 
     def self.max_jobs
-      Sidekiq.options[:dead_max_jobs]
+      Sidekiq[:dead_max_jobs]
     end
 
     def self.timeout
-      Sidekiq.options[:dead_timeout_in_seconds]
+      Sidekiq[:dead_timeout_in_seconds]
     end
   end
 

data/lib/sidekiq/cli.rb

@@ -9,18 +9,23 @@ require "erb"
 require "fileutils"
 
 require "sidekiq"
+require "sidekiq/component"
 require "sidekiq/launcher"
-require "sidekiq/util"
 
 module Sidekiq
   class CLI
-    include Util
+    include Sidekiq::Component
     include Singleton unless $TESTING
 
     attr_accessor :launcher
     attr_accessor :environment
+    attr_accessor :config
 
     def parse(args = ARGV.dup)
+      @config = Sidekiq
+      @config[:error_handlers].clear
+      @config[:error_handlers] << @config.method(:default_error_handler)
+
       setup_options(args)
       initialize_logger
       validate!
@@ -36,7 +41,7 @@ module Sidekiq
     def run(boot_app: true)
       boot_application if boot_app
 
-      if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
+      if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
         print_banner
       end
       logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
@@ -67,7 +72,7 @@ module Sidekiq
 
       # touch the connection pool so it is created before we
       # fire startup and start multithreading.
-      info = Sidekiq.redis_info
+      info = @config.redis_info
       ver = info["redis_version"]
       raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
 
@@ -85,22 +90,22 @@ module Sidekiq
 
       # Since the user can pass us a connection pool explicitly in the initializer, we
       # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
-      cursize = Sidekiq.redis_pool.size
-      needed = Sidekiq.options[:concurrency] + 2
+      cursize = @config.redis_pool.size
+      needed = @config[:concurrency] + 2
       raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
 
       # cache process identity
-      Sidekiq.options[:identity] = identity
+      @config[:identity] = identity
 
       # Touch middleware so it isn't lazy loaded by multiple threads, #3043
-      Sidekiq.server_middleware
+      @config.server_middleware
 
       # Before this point, the process is initializing with just the main thread.
       # Starting here the process will now have multiple threads running.
       fire_event(:startup, reverse: false, reraise: true)
 
-      logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(", ")}" }
-      logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(", ")}" }
+      logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
+      logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
 
       launch(self_read)
     end
@@ -110,7 +115,7 @@ module Sidekiq
         logger.info "Starting processing, hit Ctrl-C to stop"
       end
 
-      @launcher = Sidekiq::Launcher.new(options)
+      @launcher = Sidekiq::Launcher.new(@config)
 
       begin
         launcher.run
@@ -173,25 +178,25 @@ module Sidekiq
       # Heroku sends TERM and then waits 30 seconds for process to exit.
       "TERM" => ->(cli) { raise Interrupt },
       "TSTP" => ->(cli) {
-        Sidekiq.logger.info "Received TSTP, no longer accepting new work"
+        cli.logger.info "Received TSTP, no longer accepting new work"
         cli.launcher.quiet
       },
       "TTIN" => ->(cli) {
         Thread.list.each do |thread|
-          Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
+          cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
           if thread.backtrace
-            Sidekiq.logger.warn thread.backtrace.join("\n")
+            cli.logger.warn thread.backtrace.join("\n")
           else
-            Sidekiq.logger.warn "<no backtrace available>"
+            cli.logger.warn "<no backtrace available>"
           end
         end
       }
     }
-    UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
+    UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
     SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
 
     def handle_signal(sig)
-      Sidekiq.logger.debug "Got #{sig} signal"
+      logger.debug "Got #{sig} signal"
       SIGNAL_HANDLERS[sig].call(self)
     end
 
@@ -237,7 +242,7 @@ module Sidekiq
         config_dir = if File.directory?(opts[:require].to_s)
           File.join(opts[:require], "config")
         else
-          File.join(options[:require], "config")
+          File.join(@config[:require], "config")
         end
 
         %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
@@ -254,27 +259,23 @@ module Sidekiq
       opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
 
       # merge with defaults
-      options.merge!(opts)
-    end
-
-    def options
-      Sidekiq.options
+      @config.merge!(opts)
     end
 
     def boot_application
       ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
 
-      if File.directory?(options[:require])
+      if File.directory?(@config[:require])
         require "rails"
         if ::Rails::VERSION::MAJOR < 5
           raise "Sidekiq no longer supports this version of Rails"
         else
           require "sidekiq/rails"
-          require File.expand_path("#{options[:require]}/config/environment.rb")
+          require File.expand_path("#{@config[:require]}/config/environment.rb")
         end
-        options[:tag] ||= default_tag
+        @config[:tag] ||= default_tag
       else
-        require options[:require]
+        require @config[:require]
       end
     end
 
@@ -291,8 +292,8 @@ module Sidekiq
     end
 
     def validate!
-      if !File.exist?(options[:require]) ||
-          (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
+      if !File.exist?(@config[:require]) ||
+          (File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
         logger.info "=================================================================="
         logger.info "  Please point Sidekiq to a Rails application or a Ruby file  "
         logger.info "  to load your job classes with -r [DIR|FILE]."
@@ -302,7 +303,7 @@ module Sidekiq
       end
 
       [:concurrency, :timeout].each do |opt|
-        raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.key?(opt) && options[opt].to_i <= 0
+        raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
       end
     end
 
@@ -376,7 +377,7 @@ module Sidekiq
     end
 
     def initialize_logger
-      Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
+      @config.logger.level = ::Logger::DEBUG if @config[:verbose]
     end
 
     def parse_config(path)

data/lib/sidekiq/client.rb

@@ -71,7 +71,7 @@ module Sidekiq
     #
     def push(item)
       normed = normalize_item(item)
-      payload = middleware.invoke(normed["class"], normed, normed["queue"], @redis_pool) do
+      payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
         normed
       end
       if payload
@@ -110,7 +110,7 @@ module Sidekiq
       payloads = args.map.with_index { |job_args, index|
         copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
         copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
-        result = middleware.invoke(copy["class"], copy, copy["queue"], @redis_pool) do
+        result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
           verify_json(copy)
           copy
         end
@@ -201,7 +201,7 @@ module Sidekiq
           conn.pipelined do |pipeline|
             atomic_push(pipeline, payloads)
           end
-        rescue Redis::BaseError => ex
+        rescue RedisConnection.adapter::BaseError => ex
           # 2550 Failover can cause the server to become a replica, need
           # to disconnect and reopen the socket to get back to the primary.
           # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
@@ -220,7 +220,7 @@ module Sidekiq
 
     def atomic_push(conn, payloads)
       if payloads.first.key?("at")
-        conn.zadd("schedule", payloads.map { |hash|
+        conn.zadd("schedule", *payloads.map { |hash|
           at = hash.delete("at").to_s
           [at, Sidekiq.dump_json(hash)]
         })

data/lib/sidekiq/component.rb

@@ -0,0 +1,64 @@
+module Sidekiq
+  ##
+  # Sidekiq::Component assumes a config instance is available at @config
+  module Component
+    attr_reader :config
+
+    def watchdog(last_words)
+      yield
+    rescue Exception => ex
+      handle_exception(ex, {context: last_words})
+      raise ex
+    end
+
+    def safe_thread(name, &block)
+      Thread.new do
+        Thread.current.name = name
+        watchdog(name, &block)
+      end
+    end
+
+    def logger
+      config.logger
+    end
+
+    def redis(&block)
+      config.redis(&block)
+    end
+
+    def tid
+      Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
+    end
+
+    def hostname
+      ENV["DYNO"] || Socket.gethostname
+    end
+
+    def process_nonce
+      @@process_nonce ||= SecureRandom.hex(6)
+    end
+
+    def identity
+      @@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
+    end
+
+    def handle_exception(ex, ctx = {})
+      config.handle_exception(ex, ctx)
+    end
+
+    def fire_event(event, options = {})
+      reverse = options[:reverse]
+      reraise = options[:reraise]
+
+      arr = config[:lifecycle_events][event]
+      arr.reverse! if reverse
+      arr.each do |block|
+        block.call
+      rescue => ex
+        handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
+        raise ex if reraise
+      end
+      arr.clear # once we've fired an event, we never fire it again
+    end
+  end
+end

data/lib/sidekiq/fetch.rb

@@ -1,14 +1,16 @@
 # frozen_string_literal: true
 
 require "sidekiq"
+require "sidekiq/component"
 
 module Sidekiq
   class BasicFetch
+    include Sidekiq::Component
     # We want the fetch operation to timeout every few seconds so the thread
     # can check if the process is shutting down.
     TIMEOUT = 2
 
-    UnitOfWork = Struct.new(:queue, :job) {
+    UnitOfWork = Struct.new(:queue, :job, :config) {
       def acknowledge
         # nothing to do
       end
@@ -18,17 +20,17 @@ module Sidekiq
       end
 
       def requeue
-        Sidekiq.redis do |conn|
+        config.redis do |conn|
           conn.rpush(queue, job)
         end
       end
     }
 
-    def initialize(options)
-      raise ArgumentError, "missing queue list" unless options[:queues]
-      @options = options
-      @strictly_ordered_queues = !!@options[:strict]
-      @queues = @options[:queues].map { |q| "queue:#{q}" }
+    def initialize(config)
+      raise ArgumentError, "missing queue list" unless config[:queues]
+      @config = config
+      @strictly_ordered_queues = !!@config[:strict]
+      @queues = @config[:queues].map { |q| "queue:#{q}" }
       if @strictly_ordered_queues
         @queues.uniq!
         @queues << TIMEOUT
@@ -44,30 +46,30 @@ module Sidekiq
         return nil
       end
 
-      work = Sidekiq.redis { |conn| conn.brpop(*qs) }
-      UnitOfWork.new(*work) if work
+      queue, job = redis { |conn| conn.brpop(*qs) }
+      UnitOfWork.new(queue, job, config) if queue
     end
 
     def bulk_requeue(inprogress, options)
       return if inprogress.empty?
 
-      Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+      logger.debug { "Re-queueing terminated jobs" }
       jobs_to_requeue = {}
       inprogress.each do |unit_of_work|
         jobs_to_requeue[unit_of_work.queue] ||= []
         jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
       end
 
-      Sidekiq.redis do |conn|
+      redis do |conn|
         conn.pipelined do |pipeline|
           jobs_to_requeue.each do |queue, jobs|
             pipeline.rpush(queue, jobs)
           end
         end
       end
-      Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
+      logger.info("Pushed #{inprogress.size} jobs back to Redis")
     rescue => ex
-      Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
+      logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
     end
 
     # Creating the Redis#brpop command takes into account any

data/lib/sidekiq/job_retry.rb

@@ -66,12 +66,13 @@ module Sidekiq
 
     class Skip < Handled; end
 
-    include Sidekiq::Util
+    include Sidekiq::Component
 
     DEFAULT_MAX_RETRY_ATTEMPTS = 25
 
-    def initialize(options = {})
-      @max_retries = Sidekiq.options.merge(options).fetch(:max_retries, DEFAULT_MAX_RETRY_ATTEMPTS)
+    def initialize(options)
+      @config = options
+      @max_retries = @config[:max_retries] || DEFAULT_MAX_RETRY_ATTEMPTS
     end
 
     # The global retry handler requires only the barest of data.

data/lib/sidekiq/job_util.rb

@@ -4,7 +4,8 @@ require "time"
 module Sidekiq
   module JobUtil
     # These functions encapsulate various job utilities.
-    # They must be simple and free from side effects.
+
+    TRANSIENT_ATTRIBUTES = %w[]
 
     def validate(item)
       raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
@@ -16,13 +17,13 @@ module Sidekiq
 
     def verify_json(item)
       job_class = item["wrapped"] || item["class"]
-      if Sidekiq.options[:on_complex_arguments] == :raise
+      if Sidekiq[:on_complex_arguments] == :raise
         msg = <<~EOM
           Job arguments to #{job_class} must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices.
           To disable this error, remove `Sidekiq.strict_args!` from your initializer.
         EOM
         raise(ArgumentError, msg) unless json_safe?(item)
-      elsif Sidekiq.options[:on_complex_arguments] == :warn
+      elsif Sidekiq[:on_complex_arguments] == :warn
         Sidekiq.logger.warn <<~EOM unless json_safe?(item)
           Job arguments to #{job_class} do not serialize to JSON safely. This will raise an error in
           Sidekiq 7.0. See https://github.com/mperham/sidekiq/wiki/Best-Practices or raise an error today
@@ -42,6 +43,9 @@ module Sidekiq
 
       raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
 
+      # remove job attributes which aren't necessary to persist into Redis
+      TRANSIENT_ATTRIBUTES.each { |key| item.delete(key) }
+
       item["jid"] ||= SecureRandom.hex(12)
       item["class"] = item["class"].to_s
       item["queue"] = item["queue"].to_s