sidekiq 6.5.12 → 7.1.5

data/lib/sidekiq/middleware/current_attributes.rb

@@ -7,27 +7,31 @@ module Sidekiq
   # This can be useful for multi-tenancy, i18n locale, timezone, any implicit
   # per-request attribute. See +ActiveSupport::CurrentAttributes+.
   #
+  # For multiple current attributes, pass an array of current attributes.
+  #
   # @example
   #
   #   # in your initializer
   #   require "sidekiq/middleware/current_attributes"
   #   Sidekiq::CurrentAttributes.persist("Myapp::Current")
+  #   # or multiple current attributes
+  #   Sidekiq::CurrentAttributes.persist(["Myapp::Current", "Myapp::OtherCurrent"])
   #
   module CurrentAttributes
     class Save
       include Sidekiq::ClientMiddleware
 
-      def initialize(cattr)
-        @strklass = cattr
+      def initialize(cattrs)
+        @cattrs = cattrs
       end
 
       def call(_, job, _, _)
-        attrs = @strklass.constantize.attributes
-        if attrs.any?
-          if job.has_key?("cattr")
-            job["cattr"].merge!(attrs)
-          else
-            job["cattr"] = attrs
+        @cattrs.each do |(key, strklass)|
+          if !job.has_key?(key)
+            attrs = strklass.constantize.attributes
+            # Retries can push the job N times, we don't
+            # want retries to reset cattr. #5692, #5090
+            job[key] = attrs if attrs.any?
           end
         end
         yield
@@ -37,26 +41,54 @@ module Sidekiq
     class Load
       include Sidekiq::ServerMiddleware
 
-      def initialize(cattr)
-        @strklass = cattr
+      def initialize(cattrs)
+        @cattrs = cattrs
       end
 
       def call(_, job, _, &block)
-        if job.has_key?("cattr")
-          @strklass.constantize.set(job["cattr"], &block)
-        else
-          yield
+        cattrs_to_reset = []
+
+        @cattrs.each do |(key, strklass)|
+          if job.has_key?(key)
+            constklass = strklass.constantize
+            cattrs_to_reset << constklass
+
+            job[key].each do |(attribute, value)|
+              constklass.public_send("#{attribute}=", value)
+            end
+          end
         end
+
+        yield
+      ensure
+        cattrs_to_reset.each(&:reset)
       end
     end
 
-    def self.persist(klass)
-      Sidekiq.configure_client do |config|
-        config.client_middleware.add Save, klass.to_s
+    class << self
+      def persist(klass_or_array, config = Sidekiq.default_configuration)
+        cattrs = build_cattrs_hash(klass_or_array)
+
+        config.client_middleware.add Save, cattrs
+        config.server_middleware.add Load, cattrs
       end
-      Sidekiq.configure_server do |config|
-        config.client_middleware.add Save, klass.to_s
-        config.server_middleware.add Load, klass.to_s
+
+      private
+
+      def build_cattrs_hash(klass_or_array)
+        if klass_or_array.is_a?(Array)
+          {}.tap do |hash|
+            klass_or_array.each_with_index do |klass, index|
+              hash[key_at(index)] = klass.to_s
+            end
+          end
+        else
+          {key_at(0) => klass_or_array.to_s}
+        end
+      end
+
+      def key_at(index)
+        (index == 0) ? "cattr" : "cattr_#{index}"
       end
     end
   end

data/lib/sidekiq/monitor.rb

@@ -16,8 +16,6 @@ class Sidekiq::Monitor
         return
       end
       send(section)
-    rescue => e
-      abort "Couldn't get status: #{e}"
     end
 
     def all
@@ -49,10 +47,25 @@ class Sidekiq::Monitor
     def processes
       puts "---- Processes (#{process_set.size}) ----"
       process_set.each_with_index do |process, index|
+        # Keep compatibility with legacy versions since we don't want to break sidekiqmon during rolling upgrades or downgrades.
+        #
+        # Before:
+        #   ["default", "critical"]
+        #
+        # After:
+        #   {"default" => 1, "critical" => 10}
+        queues =
+          if process["weights"]
+            process["weights"].sort_by { |queue| queue[0] }.map { |capsule| capsule.map { |name, weight| (weight > 0) ? "#{name}: #{weight}" : name }.join(", ") }
+          else
+            process["queues"].sort
+          end
+
         puts "#{process["identity"]} #{tags_for(process)}"
         puts "  Started: #{Time.at(process["started_at"])} (#{time_ago(process["started_at"])})"
         puts "  Threads: #{process["concurrency"]} (#{process["busy"]} busy)"
-        puts "   Queues: #{split_multiline(process["queues"].sort, pad: 11)}"
+        puts "   Queues: #{split_multiline(queues, pad: 11)}"
+        puts "  Version: #{process["version"] || "Unknown"}" if process["version"] != Sidekiq::VERSION
         puts "" unless (index + 1) == process_set.size
       end
     end

data/lib/sidekiq/paginator.rb

@@ -19,7 +19,7 @@ module Sidekiq
           total_size, items = conn.multi { |transaction|
             transaction.zcard(key)
             if rev
-              transaction.zrevrange(key, starting, ending, withscores: true)
+              transaction.zrange(key, starting, ending, "REV", withscores: true)
             else
               transaction.zrange(key, starting, ending, withscores: true)
             end

data/lib/sidekiq/processor.rb

@@ -26,18 +26,18 @@ module Sidekiq
 
     attr_reader :thread
     attr_reader :job
+    attr_reader :capsule
 
-    def initialize(options, &block)
+    def initialize(capsule, &block)
+      @config = @capsule = capsule
       @callback = block
       @down = false
       @done = false
       @job = nil
       @thread = nil
-      @config = options
-      @strategy = options[:fetch]
-      @reloader = options[:reloader] || proc { |&block| block.call }
-      @job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
-      @retrier = Sidekiq::JobRetry.new(options)
+      @reloader = Sidekiq.default_configuration[:reloader]
+      @job_logger = (capsule.config[:job_logger] || Sidekiq::JobLogger).new(logger)
+      @retrier = Sidekiq::JobRetry.new(capsule)
     end
 
     def terminate(wait = false)
@@ -59,12 +59,16 @@ module Sidekiq
     end
 
     def start
-      @thread ||= safe_thread("processor", &method(:run))
+      @thread ||= safe_thread("#{config.name}/processor", &method(:run))
     end
 
     private unless $TESTING
 
     def run
+      # By setting this thread-local, Sidekiq.redis will access +Sidekiq::Capsule#redis_pool+
+      # instead of the global pool in +Sidekiq::Config#redis_pool+.
+      Thread.current[:sidekiq_capsule] = @capsule
+
       process_one until @done
       @callback.call(self)
     rescue Sidekiq::Shutdown
@@ -80,7 +84,7 @@ module Sidekiq
     end
 
     def get_one
-      uow = @strategy.retrieve_work
+      uow = capsule.fetcher.retrieve_work
       if @down
         logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
         @down = nil
@@ -129,7 +133,7 @@ module Sidekiq
               # the Reloader.  It handles code loading, db connection management, etc.
               # Effectively this block denotes a "unit of work" to Rails.
               @reloader.call do
-                klass = constantize(job_hash["class"])
+                klass = Object.const_get(job_hash["class"])
                 inst = klass.new
                 inst.jid = job_hash["jid"]
                 @retrier.local(inst, jobstr, queue) do
@@ -142,6 +146,11 @@ module Sidekiq
       end
     end
 
+    IGNORE_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :never}
+    private_constant :IGNORE_SHUTDOWN_INTERRUPTS
+    ALLOW_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :immediate}
+    private_constant :ALLOW_SHUTDOWN_INTERRUPTS
+
     def process(uow)
       jobstr = uow.job
       queue = uow.queue_name
@@ -153,47 +162,46 @@ module Sidekiq
       rescue => ex
         handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
         now = Time.now.to_f
-        config.redis do |conn|
+        redis do |conn|
           conn.multi do |xa|
             xa.zadd("dead", now.to_s, jobstr)
-            xa.zremrangebyscore("dead", "-inf", now - config[:dead_timeout_in_seconds])
-            xa.zremrangebyrank("dead", 0, - config[:dead_max_jobs])
+            xa.zremrangebyscore("dead", "-inf", now - @capsule.config[:dead_timeout_in_seconds])
+            xa.zremrangebyrank("dead", 0, - @capsule.config[:dead_max_jobs])
           end
         end
         return uow.acknowledge
       end
 
       ack = false
-      begin
-        dispatch(job_hash, queue, jobstr) do |inst|
-          @config.server_middleware.invoke(inst, job_hash, queue) do
-            execute_job(inst, job_hash["args"])
+      Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) do
+        Thread.handle_interrupt(ALLOW_SHUTDOWN_INTERRUPTS) do
+          dispatch(job_hash, queue, jobstr) do |inst|
+            config.server_middleware.invoke(inst, job_hash, queue) do
+              execute_job(inst, job_hash["args"])
+            end
           end
+          ack = true
+        rescue Sidekiq::Shutdown
+          # Had to force kill this job because it didn't finish
+          # within the timeout.  Don't acknowledge the work since
+          # we didn't properly finish it.
+        rescue Sidekiq::JobRetry::Handled => h
+          # this is the common case: job raised error and Sidekiq::JobRetry::Handled
+          # signals that we created a retry successfully.  We can acknowlege the job.
+          ack = true
+          e = h.cause || h
+          handle_exception(e, {context: "Job raised exception", job: job_hash})
+          raise e
+        rescue Exception => ex
+          # Unexpected error!  This is very bad and indicates an exception that got past
+          # the retry subsystem (e.g. network partition).  We won't acknowledge the job
+          # so it can be rescued when using Sidekiq Pro.
+          handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
+          raise ex
         end
-        ack = true
-      rescue Sidekiq::Shutdown
-        # Had to force kill this job because it didn't finish
-        # within the timeout.  Don't acknowledge the work since
-        # we didn't properly finish it.
-      rescue Sidekiq::JobRetry::Handled => h
-        # this is the common case: job raised error and Sidekiq::JobRetry::Handled
-        # signals that we created a retry successfully.  We can acknowlege the job.
-        ack = true
-        e = h.cause || h
-        handle_exception(e, {context: "Job raised exception", job: job_hash})
-        raise e
-      rescue Exception => ex
-        # Unexpected error!  This is very bad and indicates an exception that got past
-        # the retry subsystem (e.g. network partition).  We won't acknowledge the job
-        # so it can be rescued when using Sidekiq Pro.
-        handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
-        raise ex
       ensure
         if ack
-          # We don't want a shutdown signal to interrupt job acknowledgment.
-          Thread.handle_interrupt(Sidekiq::Shutdown => :never) do
-            uow.acknowledge
-          end
+          uow.acknowledge
         end
       end
     end
@@ -269,18 +277,5 @@ module Sidekiq
         PROCESSED.incr
       end
     end
-
-    def constantize(str)
-      return Object.const_get(str) unless str.include?("::")
-
-      names = str.split("::")
-      names.shift if names.empty? || names.first.empty?
-
-      names.inject(Object) do |constant, name|
-        # the false flag limits search for name to under the constant namespace
-        #   which mimics Rails' behaviour
-        constant.const_get(name, false)
-      end
-    end
   end
 end

data/lib/sidekiq/rails.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "sidekiq/job"
+require "rails"
 
 module Sidekiq
   class Rails < ::Rails::Engine
@@ -10,7 +11,8 @@ module Sidekiq
       end
 
       def call
-        @app.reloader.wrap do
+        params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
+        @app.reloader.wrap(**params) do
           yield
         end
       end
@@ -22,7 +24,7 @@ module Sidekiq
 
     # By including the Options module, we allow AJs to directly control sidekiq features
     # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
-    # AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
+    # AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
     # manually retried, don't automatically die, etc.
     #
     #   class SomeJob < ActiveJob::Base
@@ -37,11 +39,10 @@ module Sidekiq
       end
     end
 
-    config.before_configuration do
-      dep = ActiveSupport::Deprecation.new("7.0", "Sidekiq")
-      dep.deprecate_methods(Sidekiq.singleton_class,
-        default_worker_options: :default_job_options,
-        "default_worker_options=": :default_job_options=)
+    initializer "sidekiq.backtrace_cleaner" do
+      Sidekiq.configure_server do |config|
+        config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
+      end
     end
 
     # This hook happens after all initializers are run, just before returning

data/lib/sidekiq/redis_client_adapter.rb

@@ -1,15 +1,17 @@
 # frozen_string_literal: true
 
-require "connection_pool"
+require "set"
 require "redis_client"
 require "redis_client/decorator"
-require "uri"
 
 module Sidekiq
   class RedisClientAdapter
     BaseError = RedisClient::Error
     CommandError = RedisClient::CommandError
 
+    # You can add/remove items or clear the whole thing if you don't want deprecation warnings.
+    DEPRECATED_COMMANDS = %i[rpoplpush zrangebyscore zrevrange zrevrangebyscore getset hmset setex setnx].to_set
+
     module CompatMethods
       def info
         @client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
@@ -19,30 +21,12 @@ module Sidekiq
         @client.call("EVALSHA", sha, keys.size, *keys, *argv)
       end
 
-      def brpoplpush(*args)
-        @client.blocking_call(false, "BRPOPLPUSH", *args)
-      end
-
-      def brpop(*args)
-        @client.blocking_call(false, "BRPOP", *args)
-      end
-
-      def set(*args)
-        @client.call("SET", *args) { |r| r == "OK" }
-      end
-      ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
-
-      def sismember(*args)
-        @client.call("SISMEMBER", *args) { |c| c > 0 }
-      end
-
-      def exists?(key)
-        @client.call("EXISTS", key) { |c| c > 0 }
-      end
-
       private
 
+      # this allows us to use methods like `conn.hmset(...)` instead of having to use
+      # redis-client's native `conn.call("hmset", ...)`
       def method_missing(*args, &block)
+        warn("[sidekiq#5788] Redis has deprecated the `#{args.first}`command, called at #{caller(1..1)}") if DEPRECATED_COMMANDS.include?(args.first)
         @client.call(*args, *block)
       end
       ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
@@ -55,47 +39,8 @@ module Sidekiq
     CompatClient = RedisClient::Decorator.create(CompatMethods)
 
     class CompatClient
-      %i[scan sscan zscan hscan].each do |method|
-        alias_method :"#{method}_each", method
-        undef_method method
-      end
-
-      def disconnect!
-        @client.close
-      end
-
-      def connection
-        {id: @client.id}
-      end
-
-      def redis
-        self
-      end
-
-      def _client
-        @client
-      end
-
-      def message
-        yield nil, @queue.pop
-      end
-
-      # NB: this method does not return
-      def subscribe(chan)
-        @queue = ::Queue.new
-
-        pubsub = @client.pubsub
-        pubsub.call("subscribe", chan)
-
-        loop do
-          evt = pubsub.next_event
-          next if evt.nil?
-          next unless evt[0] == "message" && evt[1] == chan
-
-          (_, _, msg) = evt
-          @queue << msg
-          yield self
-        end
+      def config
+        @client.config
       end
     end
 
@@ -118,9 +63,7 @@ module Sidekiq
       opts = options.dup
 
       if opts[:namespace]
-        Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
-          "Either use the redis adapter or remove the namespace.")
-        Kernel.exit(-127)
+        raise ArgumentError, "Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature is no longer supported in Sidekiq 7+. See https://github.com/sidekiq/sidekiq/blob/main/docs/7.0-Upgrade.md#redis-namespace."
       end
 
       opts.delete(:size)
@@ -150,5 +93,3 @@ module Sidekiq
     end
   end
 end
-
-Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter

data/lib/sidekiq/redis_connection.rb

@@ -1,128 +1,33 @@
 # frozen_string_literal: true
 
 require "connection_pool"
-require "redis"
 require "uri"
+require "sidekiq/redis_client_adapter"
 
 module Sidekiq
   module RedisConnection
-    class RedisAdapter
-      BaseError = Redis::BaseError
-      CommandError = Redis::CommandError
-
-      def initialize(options)
-        warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
-        @options = options
-      end
-
-      def new_client
-        namespace = @options[:namespace]
-
-        client = Redis.new client_opts(@options)
-        if namespace
-          begin
-            require "redis/namespace"
-            Redis::Namespace.new(namespace, redis: client)
-          rescue LoadError
-            Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
-                                 "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
-            exit(-127)
-          end
-        else
-          client
-        end
-      end
-
-      private
-
-      def client_opts(options)
-        opts = options.dup
-        if opts[:namespace]
-          opts.delete(:namespace)
-        end
-
-        if opts[:network_timeout]
-          opts[:timeout] = opts[:network_timeout]
-          opts.delete(:network_timeout)
-        end
-
-        # Issue #3303, redis-rb will silently retry an operation.
-        # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
-        # is performed twice but I believe this is much, much rarer
-        # than the reconnect silently fixing a problem; we keep it
-        # on by default.
-        opts[:reconnect_attempts] ||= 1
-
-        opts
-      end
-    end
-
-    @adapter = RedisAdapter
-
     class << self
-      attr_reader :adapter
-
-      # RedisConnection.adapter = :redis
-      # RedisConnection.adapter = :redis_client
-      def adapter=(adapter)
-        raise "no" if adapter == self
-        result = case adapter
-        when :redis
-          RedisAdapter
-        when Class
-          adapter
-        else
-          require "sidekiq/#{adapter}_adapter"
-          nil
-        end
-        @adapter = result if result
-      end
-
       def create(options = {})
         symbolized_options = options.transform_keys(&:to_sym)
+        symbolized_options[:url] ||= determine_redis_provider
 
-        if !symbolized_options[:url] && (u = determine_redis_provider)
-          symbolized_options[:url] = u
-        end
-
-        size = if symbolized_options[:size]
-          symbolized_options[:size]
-        elsif Sidekiq.server?
-          # Give ourselves plenty of connections.  pool is lazy
-          # so we won't create them until we need them.
-          Sidekiq[:concurrency] + 5
-        elsif ENV["RAILS_MAX_THREADS"]
-          Integer(ENV["RAILS_MAX_THREADS"])
-        else
-          5
-        end
-
-        verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
+        logger = symbolized_options.delete(:logger)
+        logger&.info { "Sidekiq #{Sidekiq::VERSION} connecting to Redis with options #{scrub(symbolized_options)}" }
 
-        pool_timeout = symbolized_options[:pool_timeout] || 1
-        log_info(symbolized_options)
+        raise "Sidekiq 7+ does not support Redis protocol 2" if symbolized_options[:protocol] == 2
+        size = symbolized_options.delete(:size) || 5
+        pool_timeout = symbolized_options.delete(:pool_timeout) || 1
+        pool_name = symbolized_options.delete(:pool_name)
 
-        redis_config = adapter.new(symbolized_options)
-        ConnectionPool.new(timeout: pool_timeout, size: size) do
+        redis_config = Sidekiq::RedisClientAdapter.new(symbolized_options)
+        ConnectionPool.new(timeout: pool_timeout, size: size, name: pool_name) do
           redis_config.new_client
         end
       end
 
       private
 
-      # Sidekiq needs many concurrent Redis connections.
-      #
-      # We need a connection for each Processor.
-      # We need a connection for Pro's real-time change listener
-      # We need a connection to various features to call Redis every few seconds:
-      #   - the process heartbeat.
-      #   - enterprise's leader election
-      #   - enterprise's cron support
-      def verify_sizing(size, concurrency)
-        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
-      end
-
-      def log_info(options)
+      def scrub(options)
         redacted = "REDACTED"
 
         # Deep clone so we can muck with these options all we want and exclude
@@ -140,11 +45,7 @@ module Sidekiq
         scrubbed_options[:sentinels]&.each do |sentinel|
           sentinel[:password] = redacted if sentinel[:password]
         end
-        if Sidekiq.server?
-          Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
-        else
-          Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
-        end
+        scrubbed_options
       end
 
       def determine_redis_provider