sidekiq 6.5.12 → 7.3.9

data/lib/sidekiq/redis_client_adapter.rb

@@ -1,15 +1,17 @@
 # frozen_string_literal: true
 
-require "connection_pool"
+require "set"
 require "redis_client"
 require "redis_client/decorator"
-require "uri"
 
 module Sidekiq
   class RedisClientAdapter
     BaseError = RedisClient::Error
     CommandError = RedisClient::CommandError
 
+    # You can add/remove items or clear the whole thing if you don't want deprecation warnings.
+    DEPRECATED_COMMANDS = %i[rpoplpush zrangebyscore zrevrange zrevrangebyscore getset hmset setex setnx].to_set
+
     module CompatMethods
       def info
         @client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
@@ -19,30 +21,28 @@ module Sidekiq
         @client.call("EVALSHA", sha, keys.size, *keys, *argv)
       end
 
-      def brpoplpush(*args)
-        @client.blocking_call(false, "BRPOPLPUSH", *args)
-      end
-
-      def brpop(*args)
-        @client.blocking_call(false, "BRPOP", *args)
-      end
-
-      def set(*args)
-        @client.call("SET", *args) { |r| r == "OK" }
-      end
-      ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
-
-      def sismember(*args)
-        @client.call("SISMEMBER", *args) { |c| c > 0 }
-      end
+      # this is the set of Redis commands used by Sidekiq. Not guaranteed
+      # to be comprehensive, we use this as a performance enhancement to
+      # avoid calling method_missing on most commands
+      USED_COMMANDS = %w[bitfield bitfield_ro del exists expire flushdb
+        get hdel hget hgetall hincrby hlen hmget hset hsetnx incr incrby
+        lindex llen lmove lpop lpush lrange lrem mget mset ping pttl
+        publish rpop rpush sadd scard script set sismember smembers
+        srem ttl type unlink zadd zcard zincrby zrange zrem
+        zremrangebyrank zremrangebyscore]
 
-      def exists?(key)
-        @client.call("EXISTS", key) { |c| c > 0 }
+      USED_COMMANDS.each do |name|
+        define_method(name) do |*args, **kwargs|
+          @client.call(name, *args, **kwargs)
+        end
       end
 
       private
 
+      # this allows us to use methods like `conn.hmset(...)` instead of having to use
+      # redis-client's native `conn.call("hmset", ...)`
       def method_missing(*args, &block)
+        warn("[sidekiq#5788] Redis has deprecated the `#{args.first}`command, called at #{caller(1..1)}") if DEPRECATED_COMMANDS.include?(args.first)
         @client.call(*args, *block)
       end
       ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
@@ -55,47 +55,8 @@ module Sidekiq
     CompatClient = RedisClient::Decorator.create(CompatMethods)
 
     class CompatClient
-      %i[scan sscan zscan hscan].each do |method|
-        alias_method :"#{method}_each", method
-        undef_method method
-      end
-
-      def disconnect!
-        @client.close
-      end
-
-      def connection
-        {id: @client.id}
-      end
-
-      def redis
-        self
-      end
-
-      def _client
-        @client
-      end
-
-      def message
-        yield nil, @queue.pop
-      end
-
-      # NB: this method does not return
-      def subscribe(chan)
-        @queue = ::Queue.new
-
-        pubsub = @client.pubsub
-        pubsub.call("subscribe", chan)
-
-        loop do
-          evt = pubsub.next_event
-          next if evt.nil?
-          next unless evt[0] == "message" && evt[1] == chan
-
-          (_, _, msg) = evt
-          @queue << msg
-          yield self
-        end
+      def config
+        @client.config
       end
     end
 
@@ -103,6 +64,13 @@ module Sidekiq
       opts = client_opts(options)
       @config = if opts.key?(:sentinels)
         RedisClient.sentinel(**opts)
+      elsif opts.key?(:nodes)
+        # Sidekiq does not support Redis clustering but Sidekiq Enterprise's
+        # rate limiters are cluster-safe so we can scale to millions
+        # of rate limiters using a Redis cluster. This requires the
+        # `redis-cluster-client` gem.
+        # Sidekiq::Limiter.redis = { nodes: [...] }
+        RedisClient.cluster(**opts)
       else
         RedisClient.config(**opts)
       end
@@ -118,9 +86,7 @@ module Sidekiq
       opts = options.dup
 
       if opts[:namespace]
-        Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
-          "Either use the redis adapter or remove the namespace.")
-        Kernel.exit(-127)
+        raise ArgumentError, "Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature is no longer supported in Sidekiq 7+. See https://github.com/sidekiq/sidekiq/blob/main/docs/7.0-Upgrade.md#redis-namespace."
       end
 
       opts.delete(:size)
@@ -131,13 +97,9 @@ module Sidekiq
         opts.delete(:network_timeout)
       end
 
-      if opts[:driver]
-        opts[:driver] = opts[:driver].to_sym
-      end
-
       opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
       opts[:role] = opts[:role].to_sym if opts.key?(:role)
-      opts.delete(:url) if opts.key?(:sentinels)
+      opts[:driver] = opts[:driver].to_sym if opts.key?(:driver)
 
       # Issue #3303, redis-rb will silently retry an operation.
       # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
@@ -150,5 +112,3 @@ module Sidekiq
     end
   end
 end
-
-Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter

data/lib/sidekiq/redis_connection.rb

@@ -1,128 +1,57 @@
 # frozen_string_literal: true
 
 require "connection_pool"
-require "redis"
 require "uri"
+require "sidekiq/redis_client_adapter"
 
 module Sidekiq
   module RedisConnection
-    class RedisAdapter
-      BaseError = Redis::BaseError
-      CommandError = Redis::CommandError
-
-      def initialize(options)
-        warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
-        @options = options
-      end
-
-      def new_client
-        namespace = @options[:namespace]
-
-        client = Redis.new client_opts(@options)
-        if namespace
-          begin
-            require "redis/namespace"
-            Redis::Namespace.new(namespace, redis: client)
-          rescue LoadError
-            Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
-                                 "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
-            exit(-127)
-          end
-        else
-          client
-        end
-      end
-
-      private
-
-      def client_opts(options)
-        opts = options.dup
-        if opts[:namespace]
-          opts.delete(:namespace)
-        end
-
-        if opts[:network_timeout]
-          opts[:timeout] = opts[:network_timeout]
-          opts.delete(:network_timeout)
-        end
-
-        # Issue #3303, redis-rb will silently retry an operation.
-        # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
-        # is performed twice but I believe this is much, much rarer
-        # than the reconnect silently fixing a problem; we keep it
-        # on by default.
-        opts[:reconnect_attempts] ||= 1
-
-        opts
-      end
-    end
-
-    @adapter = RedisAdapter
-
     class << self
-      attr_reader :adapter
-
-      # RedisConnection.adapter = :redis
-      # RedisConnection.adapter = :redis_client
-      def adapter=(adapter)
-        raise "no" if adapter == self
-        result = case adapter
-        when :redis
-          RedisAdapter
-        when Class
-          adapter
-        else
-          require "sidekiq/#{adapter}_adapter"
-          nil
-        end
-        @adapter = result if result
-      end
-
       def create(options = {})
-        symbolized_options = options.transform_keys(&:to_sym)
+        symbolized_options = deep_symbolize_keys(options)
+        symbolized_options[:url] ||= determine_redis_provider
 
-        if !symbolized_options[:url] && (u = determine_redis_provider)
-          symbolized_options[:url] = u
-        end
+        logger = symbolized_options.delete(:logger)
+        logger&.info { "Sidekiq #{Sidekiq::VERSION} connecting to Redis with options #{scrub(symbolized_options)}" }
 
-        size = if symbolized_options[:size]
-          symbolized_options[:size]
-        elsif Sidekiq.server?
-          # Give ourselves plenty of connections.  pool is lazy
-          # so we won't create them until we need them.
-          Sidekiq[:concurrency] + 5
-        elsif ENV["RAILS_MAX_THREADS"]
-          Integer(ENV["RAILS_MAX_THREADS"])
-        else
-          5
-        end
+        raise "Sidekiq 7+ does not support Redis protocol 2" if symbolized_options[:protocol] == 2
+
+        safe = !!symbolized_options.delete(:cluster_safe)
+        raise ":nodes not allowed, Sidekiq is not safe to run on Redis Cluster" if !safe && symbolized_options.key?(:nodes)
 
-        verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
+        size = symbolized_options.delete(:size) || 5
+        pool_timeout = symbolized_options.delete(:pool_timeout) || 1
+        pool_name = symbolized_options.delete(:pool_name)
 
-        pool_timeout = symbolized_options[:pool_timeout] || 1
-        log_info(symbolized_options)
+        # Default timeout in redis-client is 1 second, which can be too aggressive
+        # if the Sidekiq process is CPU-bound. With 10-15 threads and a thread quantum of 100ms,
+        # it can be easy to get the occasional ReadTimeoutError. You can still provide
+        # a smaller timeout explicitly:
+        #     config.redis = { url: "...", timeout: 1 }
+        symbolized_options[:timeout] ||= 3
 
-        redis_config = adapter.new(symbolized_options)
-        ConnectionPool.new(timeout: pool_timeout, size: size) do
+        redis_config = Sidekiq::RedisClientAdapter.new(symbolized_options)
+        ConnectionPool.new(timeout: pool_timeout, size: size, name: pool_name) do
           redis_config.new_client
         end
       end
 
       private
 
-      # Sidekiq needs many concurrent Redis connections.
-      #
-      # We need a connection for each Processor.
-      # We need a connection for Pro's real-time change listener
-      # We need a connection to various features to call Redis every few seconds:
-      #   - the process heartbeat.
-      #   - enterprise's leader election
-      #   - enterprise's cron support
-      def verify_sizing(size, concurrency)
-        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
+      def deep_symbolize_keys(object)
+        case object
+        when Hash
+          object.each_with_object({}) do |(key, value), result|
+            result[key.to_sym] = deep_symbolize_keys(value)
+          end
+        when Array
+          object.map { |e| deep_symbolize_keys(e) }
+        else
+          object
+        end
       end
 
-      def log_info(options)
+      def scrub(options)
         redacted = "REDACTED"
 
         # Deep clone so we can muck with these options all we want and exclude
@@ -134,17 +63,19 @@ module Sidekiq
           uri.password = redacted
           scrubbed_options[:url] = uri.to_s
         end
-        if scrubbed_options[:password]
-          scrubbed_options[:password] = redacted
-        end
+        scrubbed_options[:password] = redacted if scrubbed_options[:password]
+        scrubbed_options[:sentinel_password] = redacted if scrubbed_options[:sentinel_password]
         scrubbed_options[:sentinels]&.each do |sentinel|
-          sentinel[:password] = redacted if sentinel[:password]
-        end
-        if Sidekiq.server?
-          Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
-        else
-          Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
+          if sentinel.is_a?(String)
+            if (uri = URI(sentinel)) && uri.password
+              uri.password = redacted
+              sentinel.replace(uri.to_s)
+            end
+          elsif sentinel[:password]
+            sentinel[:password] = redacted
+          end
         end
+        scrubbed_options
       end
 
       def determine_redis_provider
@@ -166,9 +97,7 @@ module Sidekiq
           EOM
         end
 
-        ENV[
-          p || "REDIS_URL"
-        ]
+        ENV[p.to_s] || ENV["REDIS_URL"]
       end
     end
   end

data/lib/sidekiq/ring_buffer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "forwardable"
 
 module Sidekiq

data/lib/sidekiq/scheduled.rb

@@ -8,16 +8,20 @@ module Sidekiq
     SETS = %w[retry schedule]
 
     class Enq
+      include Sidekiq::Component
+
       LUA_ZPOPBYSCORE = <<~LUA
         local key, now = KEYS[1], ARGV[1]
-        local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
+        local jobs = redis.call("zrange", key, "-inf", now, "byscore", "limit", 0, 1)
         if jobs[1] then
           redis.call("zrem", key, jobs[1])
           return jobs[1]
         end
       LUA
 
-      def initialize
+      def initialize(container)
+        @config = container
+        @client = Sidekiq::Client.new(config: container)
         @done = false
         @lua_zpopbyscore_sha = nil
       end
@@ -25,15 +29,15 @@ module Sidekiq
       def enqueue_jobs(sorted_sets = SETS)
         # A job's "score" in Redis is the time at which it should be processed.
         # Just check Redis for the set of jobs with a timestamp before now.
-        Sidekiq.redis do |conn|
+        redis do |conn|
           sorted_sets.each do |sorted_set|
             # Get next item in the queue with score (time to execute) <= now.
             # We need to go through the list one at a time to reduce the risk of something
             # going wrong between the time jobs are popped from the scheduled queue and when
             # they are pushed onto a work queue and losing the jobs.
             while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s]))
-              Sidekiq::Client.push(Sidekiq.load_json(job))
-              Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
+              @client.push(Sidekiq.load_json(job))
+              logger.debug { "enqueued #{sorted_set}: #{job}" }
             end
           end
         end
@@ -47,12 +51,11 @@ module Sidekiq
 
       def zpopbyscore(conn, keys: nil, argv: nil)
         if @lua_zpopbyscore_sha.nil?
-          raw_conn = conn.respond_to?(:redis) ? conn.redis : conn
-          @lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
+          @lua_zpopbyscore_sha = conn.script(:load, LUA_ZPOPBYSCORE)
         end
 
-        conn.evalsha(@lua_zpopbyscore_sha, keys, argv)
-      rescue RedisConnection.adapter::CommandError => e
+        conn.call("EVALSHA", @lua_zpopbyscore_sha, keys.size, *keys, *argv)
+      rescue RedisClient::CommandError => e
         raise unless e.message.start_with?("NOSCRIPT")
 
         @lua_zpopbyscore_sha = nil
@@ -70,9 +73,9 @@ module Sidekiq
 
       INITIAL_WAIT = 10
 
-      def initialize(options)
-        @config = options
-        @enq = (options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
+      def initialize(config)
+        @config = config
+        @enq = (config[:scheduled_enq] || Sidekiq::Scheduled::Enq).new(config)
         @sleeper = ConnectionPool::TimedStack.new
         @done = false
         @thread = nil
@@ -82,14 +85,10 @@ module Sidekiq
       # Shut down this instance, will pause until the thread is dead.
       def terminate
         @done = true
-        @enq.terminate if @enq.respond_to?(:terminate)
+        @enq.terminate
 
-        if @thread
-          t = @thread
-          @thread = nil
-          @sleeper << 0
-          t.value
-        end
+        @sleeper << 0
+        @thread&.value
       end
 
       def start
@@ -145,7 +144,7 @@ module Sidekiq
         # In the example above, each process should schedule every 10 seconds on average. We special
         # case smaller clusters to add 50% so they would sleep somewhere between 5 and 15 seconds.
         # As we run more processes, the scheduling interval average will approach an even spread
-        # between 0 and poll interval so we don't need this artifical boost.
+        # between 0 and poll interval so we don't need this artificial boost.
         #
         count = process_count
         interval = poll_interval_average(count)
@@ -194,11 +193,11 @@ module Sidekiq
       # should never depend on sidekiq/api.
       def cleanup
         # dont run cleanup more than once per minute
-        return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
+        return 0 unless redis { |conn| conn.set("process_cleanup", "1", "NX", "EX", "60") }
 
         count = 0
-        Sidekiq.redis do |conn|
-          procs = conn.sscan_each("processes").to_a
+        redis do |conn|
+          procs = conn.sscan("processes").to_a
           heartbeats = conn.pipelined { |pipeline|
             procs.each do |key|
               pipeline.hget(key, "info")

data/lib/sidekiq/systemd.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 #
 # Sidekiq's systemd integration allows Sidekiq to inform systemd:
 #  1. when it has successfully started

data/lib/sidekiq/testing.rb

@@ -5,23 +5,42 @@ require "sidekiq"
 
 module Sidekiq
   class Testing
+    class TestModeAlreadySetError < RuntimeError; end
     class << self
-      attr_accessor :__test_mode
+      attr_accessor :__global_test_mode
 
+      # Calling without a block sets the global test mode, affecting
+      # all threads. Calling with a block only affects the current Thread.
       def __set_test_mode(mode)
         if block_given?
-          current_mode = __test_mode
+          # Reentrant testing modes will lead to a rat's nest of code which is
+          # hard to reason about. You can set the testing mode once globally and
+          # you can override that global setting once per-thread.
+          raise TestModeAlreadySetError, "Nesting test modes is not supported" if __local_test_mode
+
+          self.__local_test_mode = mode
           begin
-            self.__test_mode = mode
             yield
           ensure
-            self.__test_mode = current_mode
+            self.__local_test_mode = nil
           end
         else
-          self.__test_mode = mode
+          self.__global_test_mode = mode
         end
       end
 
+      def __test_mode
+        __local_test_mode || __global_test_mode
+      end
+
+      def __local_test_mode
+        Thread.current[:__sidekiq_test_mode]
+      end
+
+      def __local_test_mode=(value)
+        Thread.current[:__sidekiq_test_mode] = value
+      end
+
       def disable!(&block)
         __set_test_mode(:disable, &block)
       end
@@ -51,19 +70,10 @@ module Sidekiq
       end
 
       def server_middleware
-        @server_chain ||= Middleware::Chain.new
+        @server_chain ||= Middleware::Chain.new(Sidekiq.default_configuration)
         yield @server_chain if block_given?
         @server_chain
       end
-
-      def constantize(str)
-        names = str.split("::")
-        names.shift if names.empty? || names.first.empty?
-
-        names.inject(Object) do |constant, name|
-          constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
-        end
-      end
     end
   end
 
@@ -73,7 +83,7 @@ module Sidekiq
   class EmptyQueueError < RuntimeError; end
 
   module TestingClient
-    def raw_push(payloads)
+    def atomic_push(conn, payloads)
       if Sidekiq::Testing.fake?
         payloads.each do |job|
           job = Sidekiq.load_json(Sidekiq.dump_json(job))
@@ -83,7 +93,7 @@ module Sidekiq
         true
       elsif Sidekiq::Testing.inline?
         payloads.each do |job|
-          klass = Sidekiq::Testing.constantize(job["class"])
+          klass = Object.const_get(job["class"])
           job["id"] ||= SecureRandom.hex(12)
           job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
           klass.process_job(job_hash)
@@ -102,7 +112,7 @@ module Sidekiq
     # The Queues class is only for testing the fake queue implementation.
     # There are 2 data structures involved in tandem. This is due to the
     # Rspec syntax of change(HardJob.jobs, :size). It keeps a reference
-    # to the array. Because the array was dervied from a filter of the total
+    # to the array. Because the array was derived from a filter of the total
     # jobs enqueued, it appeared as though the array didn't change.
     #
     # To solve this, we'll keep 2 hashes containing the jobs. One with keys based
@@ -218,25 +228,9 @@ module Sidekiq
     #   assert_equal 1, HardJob.jobs.size
     #   assert_equal :something, HardJob.jobs[0]['args'][0]
     #
-    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   MyMailer.delay.send_welcome_email('foo@example.com')
-    #   assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #
     # You can also clear and drain all job types:
     #
-    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
-    #
-    #   MyMailer.delay.send_welcome_email('foo@example.com')
-    #   MyModel.delay.do_something_hard
-    #
-    #   assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   assert_equal 1, Sidekiq::Extensions::DelayedModel.jobs.size
-    #
-    #   Sidekiq::Worker.clear_all # or .drain_all
-    #
-    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
+    #   Sidekiq::Job.clear_all # or .drain_all
     #
     # This can be useful to make sure jobs don't linger between tests:
     #
@@ -284,16 +278,16 @@ module Sidekiq
       def perform_one
         raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
         next_job = jobs.first
-        Queues.delete_for(next_job["jid"], queue, to_s)
+        Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
         process_job(next_job)
       end
 
       def process_job(job)
-        inst = new
-        inst.jid = job["jid"]
-        inst.bid = job["bid"] if inst.respond_to?(:bid=)
-        Sidekiq::Testing.server_middleware.invoke(inst, job, job["queue"]) do
-          execute_job(inst, job["args"])
+        instance = new
+        instance.jid = job["jid"]
+        instance.bid = job["bid"] if instance.respond_to?(:bid=)
+        Sidekiq::Testing.server_middleware.invoke(instance, job, job["queue"]) do
+          execute_job(instance, job["args"])
         end
       end
 
@@ -318,7 +312,7 @@ module Sidekiq
           job_classes = jobs.map { |job| job["class"] }.uniq
 
           job_classes.each do |job_class|
-            Sidekiq::Testing.constantize(job_class).drain
+            Object.const_get(job_class).drain
           end
         end
       end
@@ -329,13 +323,10 @@ module Sidekiq
     def jobs_for(klass)
       jobs.select do |job|
         marshalled = job["args"][0]
-        marshalled.index(klass.to_s) && YAML.load(marshalled)[0] == klass
+        marshalled.index(klass.to_s) && YAML.safe_load(marshalled)[0] == klass
       end
     end
   end
-
-  Sidekiq::Extensions::DelayedMailer.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedMailer)
-  Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
 end
 
 if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING