sidekiq 6.5.12 → 7.0.6

data/lib/sidekiq/processor.rb

@@ -26,18 +26,18 @@ module Sidekiq
 
     attr_reader :thread
     attr_reader :job
+    attr_reader :capsule
 
-    def initialize(options, &block)
+    def initialize(capsule, &block)
+      @config = @capsule = capsule
       @callback = block
       @down = false
       @done = false
       @job = nil
       @thread = nil
-      @config = options
-      @strategy = options[:fetch]
-      @reloader = options[:reloader] || proc { |&block| block.call }
-      @job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
-      @retrier = Sidekiq::JobRetry.new(options)
+      @reloader = Sidekiq.default_configuration[:reloader]
+      @job_logger = (capsule.config[:job_logger] || Sidekiq::JobLogger).new(logger)
+      @retrier = Sidekiq::JobRetry.new(capsule)
     end
 
     def terminate(wait = false)
@@ -59,12 +59,16 @@ module Sidekiq
     end
 
     def start
-      @thread ||= safe_thread("processor", &method(:run))
+      @thread ||= safe_thread("#{config.name}/processor", &method(:run))
     end
 
     private unless $TESTING
 
     def run
+      # By setting this thread-local, Sidekiq.redis will access +Sidekiq::Capsule#redis_pool+
+      # instead of the global pool in +Sidekiq::Config#redis_pool+.
+      Thread.current[:sidekiq_capsule] = @capsule
+
       process_one until @done
       @callback.call(self)
     rescue Sidekiq::Shutdown
@@ -80,7 +84,7 @@ module Sidekiq
     end
 
     def get_one
-      uow = @strategy.retrieve_work
+      uow = capsule.fetcher.retrieve_work
       if @down
         logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
         @down = nil
@@ -129,7 +133,7 @@ module Sidekiq
               # the Reloader.  It handles code loading, db connection management, etc.
               # Effectively this block denotes a "unit of work" to Rails.
               @reloader.call do
-                klass = constantize(job_hash["class"])
+                klass = Object.const_get(job_hash["class"])
                 inst = klass.new
                 inst.jid = job_hash["jid"]
                 @retrier.local(inst, jobstr, queue) do
@@ -142,6 +146,9 @@ module Sidekiq
       end
     end
 
+    IGNORE_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :never}
+    private_constant :IGNORE_SHUTDOWN_INTERRUPTS
+
     def process(uow)
       jobstr = uow.job
       queue = uow.queue_name
@@ -153,11 +160,11 @@ module Sidekiq
       rescue => ex
         handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
         now = Time.now.to_f
-        config.redis do |conn|
+        redis do |conn|
           conn.multi do |xa|
             xa.zadd("dead", now.to_s, jobstr)
-            xa.zremrangebyscore("dead", "-inf", now - config[:dead_timeout_in_seconds])
-            xa.zremrangebyrank("dead", 0, - config[:dead_max_jobs])
+            xa.zremrangebyscore("dead", "-inf", now - @capsule.config[:dead_timeout_in_seconds])
+            xa.zremrangebyrank("dead", 0, - @capsule.config[:dead_max_jobs])
           end
         end
         return uow.acknowledge
@@ -166,7 +173,7 @@ module Sidekiq
       ack = false
       begin
         dispatch(job_hash, queue, jobstr) do |inst|
-          @config.server_middleware.invoke(inst, job_hash, queue) do
+          config.server_middleware.invoke(inst, job_hash, queue) do
             execute_job(inst, job_hash["args"])
           end
         end
@@ -191,7 +198,7 @@ module Sidekiq
       ensure
         if ack
           # We don't want a shutdown signal to interrupt job acknowledgment.
-          Thread.handle_interrupt(Sidekiq::Shutdown => :never) do
+          Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) do
             uow.acknowledge
           end
         end
@@ -269,18 +276,5 @@ module Sidekiq
         PROCESSED.incr
       end
     end
-
-    def constantize(str)
-      return Object.const_get(str) unless str.include?("::")
-
-      names = str.split("::")
-      names.shift if names.empty? || names.first.empty?
-
-      names.inject(Object) do |constant, name|
-        # the false flag limits search for name to under the constant namespace
-        #   which mimics Rails' behaviour
-        constant.const_get(name, false)
-      end
-    end
   end
 end

data/lib/sidekiq/rails.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "sidekiq/job"
+require "rails"
 
 module Sidekiq
   class Rails < ::Rails::Engine
@@ -10,7 +11,8 @@ module Sidekiq
       end
 
       def call
-        @app.reloader.wrap do
+        params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
+        @app.reloader.wrap(**params) do
           yield
         end
       end
@@ -22,7 +24,7 @@ module Sidekiq
 
     # By including the Options module, we allow AJs to directly control sidekiq features
     # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
-    # AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
+    # AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
     # manually retried, don't automatically die, etc.
     #
     #   class SomeJob < ActiveJob::Base
@@ -37,11 +39,15 @@ module Sidekiq
       end
     end
 
-    config.before_configuration do
-      dep = ActiveSupport::Deprecation.new("7.0", "Sidekiq")
-      dep.deprecate_methods(Sidekiq.singleton_class,
-        default_worker_options: :default_job_options,
-        "default_worker_options=": :default_job_options=)
+    initializer "sidekiq.rails_logger" do
+      Sidekiq.configure_server do |config|
+        # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
+        # it will appear in the Sidekiq console with all of the job context. See #5021 and
+        # https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
+        unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
+          ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
+        end
+      end
     end
 
     # This hook happens after all initializers are run, just before returning
@@ -51,16 +57,6 @@ module Sidekiq
     config.after_initialize do
       Sidekiq.configure_server do |config|
         config[:reloader] = Sidekiq::Rails::Reloader.new
-
-        # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
-        # it will appear in the Sidekiq console with all of the job context.
-        unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
-          if ::Rails::VERSION::STRING < "7.1"
-            ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
-          else
-            ::Rails.logger.broadcast_to(config.logger)
-          end
-        end
       end
     end
   end

data/lib/sidekiq/redis_client_adapter.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
-require "connection_pool"
 require "redis_client"
 require "redis_client/decorator"
-require "uri"
 
 module Sidekiq
   class RedisClientAdapter
@@ -11,37 +9,20 @@ module Sidekiq
     CommandError = RedisClient::CommandError
 
     module CompatMethods
+      # TODO Deprecate and remove this
       def info
         @client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
       end
 
+      # TODO Deprecate and remove this
       def evalsha(sha, keys, argv)
         @client.call("EVALSHA", sha, keys.size, *keys, *argv)
       end
 
-      def brpoplpush(*args)
-        @client.blocking_call(false, "BRPOPLPUSH", *args)
-      end
-
-      def brpop(*args)
-        @client.blocking_call(false, "BRPOP", *args)
-      end
-
-      def set(*args)
-        @client.call("SET", *args) { |r| r == "OK" }
-      end
-      ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
-
-      def sismember(*args)
-        @client.call("SISMEMBER", *args) { |c| c > 0 }
-      end
-
-      def exists?(key)
-        @client.call("EXISTS", key) { |c| c > 0 }
-      end
-
       private
 
+      # this allows us to use methods like `conn.hmset(...)` instead of having to use
+      # redis-client's native `conn.call("hmset", ...)`
       def method_missing(*args, &block)
         @client.call(*args, *block)
       end
@@ -55,25 +36,8 @@ module Sidekiq
     CompatClient = RedisClient::Decorator.create(CompatMethods)
 
     class CompatClient
-      %i[scan sscan zscan hscan].each do |method|
-        alias_method :"#{method}_each", method
-        undef_method method
-      end
-
-      def disconnect!
-        @client.close
-      end
-
-      def connection
-        {id: @client.id}
-      end
-
-      def redis
-        self
-      end
-
-      def _client
-        @client
+      def config
+        @client.config
       end
 
       def message
@@ -118,9 +82,8 @@ module Sidekiq
       opts = options.dup
 
       if opts[:namespace]
-        Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
-          "Either use the redis adapter or remove the namespace.")
-        Kernel.exit(-127)
+        raise ArgumentError, "Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
+          "Either use the redis adapter or remove the namespace."
       end
 
       opts.delete(:size)
@@ -150,5 +113,3 @@ module Sidekiq
     end
   end
 end
-
-Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter

data/lib/sidekiq/redis_connection.rb

@@ -1,128 +1,32 @@
 # frozen_string_literal: true
 
 require "connection_pool"
-require "redis"
 require "uri"
+require "sidekiq/redis_client_adapter"
 
 module Sidekiq
   module RedisConnection
-    class RedisAdapter
-      BaseError = Redis::BaseError
-      CommandError = Redis::CommandError
-
-      def initialize(options)
-        warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
-        @options = options
-      end
-
-      def new_client
-        namespace = @options[:namespace]
-
-        client = Redis.new client_opts(@options)
-        if namespace
-          begin
-            require "redis/namespace"
-            Redis::Namespace.new(namespace, redis: client)
-          rescue LoadError
-            Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
-                                 "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
-            exit(-127)
-          end
-        else
-          client
-        end
-      end
-
-      private
-
-      def client_opts(options)
-        opts = options.dup
-        if opts[:namespace]
-          opts.delete(:namespace)
-        end
-
-        if opts[:network_timeout]
-          opts[:timeout] = opts[:network_timeout]
-          opts.delete(:network_timeout)
-        end
-
-        # Issue #3303, redis-rb will silently retry an operation.
-        # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
-        # is performed twice but I believe this is much, much rarer
-        # than the reconnect silently fixing a problem; we keep it
-        # on by default.
-        opts[:reconnect_attempts] ||= 1
-
-        opts
-      end
-    end
-
-    @adapter = RedisAdapter
-
     class << self
-      attr_reader :adapter
-
-      # RedisConnection.adapter = :redis
-      # RedisConnection.adapter = :redis_client
-      def adapter=(adapter)
-        raise "no" if adapter == self
-        result = case adapter
-        when :redis
-          RedisAdapter
-        when Class
-          adapter
-        else
-          require "sidekiq/#{adapter}_adapter"
-          nil
-        end
-        @adapter = result if result
-      end
-
       def create(options = {})
         symbolized_options = options.transform_keys(&:to_sym)
+        symbolized_options[:url] ||= determine_redis_provider
 
-        if !symbolized_options[:url] && (u = determine_redis_provider)
-          symbolized_options[:url] = u
-        end
-
-        size = if symbolized_options[:size]
-          symbolized_options[:size]
-        elsif Sidekiq.server?
-          # Give ourselves plenty of connections.  pool is lazy
-          # so we won't create them until we need them.
-          Sidekiq[:concurrency] + 5
-        elsif ENV["RAILS_MAX_THREADS"]
-          Integer(ENV["RAILS_MAX_THREADS"])
-        else
-          5
-        end
-
-        verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
+        logger = symbolized_options.delete(:logger)
+        logger&.info { "Sidekiq #{Sidekiq::VERSION} connecting to Redis with options #{scrub(symbolized_options)}" }
 
-        pool_timeout = symbolized_options[:pool_timeout] || 1
-        log_info(symbolized_options)
+        size = symbolized_options.delete(:size) || 5
+        pool_timeout = symbolized_options.delete(:pool_timeout) || 1
+        pool_name = symbolized_options.delete(:pool_name)
 
-        redis_config = adapter.new(symbolized_options)
-        ConnectionPool.new(timeout: pool_timeout, size: size) do
+        redis_config = Sidekiq::RedisClientAdapter.new(symbolized_options)
+        ConnectionPool.new(timeout: pool_timeout, size: size, name: pool_name) do
           redis_config.new_client
         end
       end
 
       private
 
-      # Sidekiq needs many concurrent Redis connections.
-      #
-      # We need a connection for each Processor.
-      # We need a connection for Pro's real-time change listener
-      # We need a connection to various features to call Redis every few seconds:
-      #   - the process heartbeat.
-      #   - enterprise's leader election
-      #   - enterprise's cron support
-      def verify_sizing(size, concurrency)
-        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
-      end
-
-      def log_info(options)
+      def scrub(options)
         redacted = "REDACTED"
 
         # Deep clone so we can muck with these options all we want and exclude
@@ -140,11 +44,7 @@ module Sidekiq
         scrubbed_options[:sentinels]&.each do |sentinel|
           sentinel[:password] = redacted if sentinel[:password]
         end
-        if Sidekiq.server?
-          Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
-        else
-          Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
-        end
+        scrubbed_options
       end
 
       def determine_redis_provider

data/lib/sidekiq/scheduled.rb

@@ -8,6 +8,8 @@ module Sidekiq
     SETS = %w[retry schedule]
 
     class Enq
+      include Sidekiq::Component
+
       LUA_ZPOPBYSCORE = <<~LUA
         local key, now = KEYS[1], ARGV[1]
         local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
@@ -17,7 +19,9 @@ module Sidekiq
         end
       LUA
 
-      def initialize
+      def initialize(container)
+        @config = container
+        @client = Sidekiq::Client.new(config: container)
         @done = false
         @lua_zpopbyscore_sha = nil
       end
@@ -25,15 +29,15 @@ module Sidekiq
       def enqueue_jobs(sorted_sets = SETS)
         # A job's "score" in Redis is the time at which it should be processed.
         # Just check Redis for the set of jobs with a timestamp before now.
-        Sidekiq.redis do |conn|
+        redis do |conn|
           sorted_sets.each do |sorted_set|
             # Get next item in the queue with score (time to execute) <= now.
             # We need to go through the list one at a time to reduce the risk of something
             # going wrong between the time jobs are popped from the scheduled queue and when
             # they are pushed onto a work queue and losing the jobs.
             while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s]))
-              Sidekiq::Client.push(Sidekiq.load_json(job))
-              Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
+              @client.push(Sidekiq.load_json(job))
+              logger.debug { "enqueued #{sorted_set}: #{job}" }
             end
           end
         end
@@ -47,12 +51,11 @@ module Sidekiq
 
       def zpopbyscore(conn, keys: nil, argv: nil)
         if @lua_zpopbyscore_sha.nil?
-          raw_conn = conn.respond_to?(:redis) ? conn.redis : conn
-          @lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
+          @lua_zpopbyscore_sha = conn.script(:load, LUA_ZPOPBYSCORE)
         end
 
-        conn.evalsha(@lua_zpopbyscore_sha, keys, argv)
-      rescue RedisConnection.adapter::CommandError => e
+        conn.call("EVALSHA", @lua_zpopbyscore_sha, keys.size, *keys, *argv)
+      rescue RedisClient::CommandError => e
         raise unless e.message.start_with?("NOSCRIPT")
 
         @lua_zpopbyscore_sha = nil
@@ -70,9 +73,9 @@ module Sidekiq
 
       INITIAL_WAIT = 10
 
-      def initialize(options)
-        @config = options
-        @enq = (options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
+      def initialize(config)
+        @config = config
+        @enq = (config[:scheduled_enq] || Sidekiq::Scheduled::Enq).new(config)
         @sleeper = ConnectionPool::TimedStack.new
         @done = false
         @thread = nil
@@ -82,14 +85,10 @@ module Sidekiq
       # Shut down this instance, will pause until the thread is dead.
       def terminate
         @done = true
-        @enq.terminate if @enq.respond_to?(:terminate)
+        @enq.terminate
 
-        if @thread
-          t = @thread
-          @thread = nil
-          @sleeper << 0
-          t.value
-        end
+        @sleeper << 0
+        @thread&.value
       end
 
       def start
@@ -194,11 +193,11 @@ module Sidekiq
       # should never depend on sidekiq/api.
       def cleanup
         # dont run cleanup more than once per minute
-        return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
+        return 0 unless redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
 
         count = 0
-        Sidekiq.redis do |conn|
-          procs = conn.sscan_each("processes").to_a
+        redis do |conn|
+          procs = conn.sscan("processes").to_a
           heartbeats = conn.pipelined { |pipeline|
             procs.each do |key|
               pipeline.hget(key, "info")

data/lib/sidekiq/testing.rb

@@ -51,19 +51,10 @@ module Sidekiq
       end
 
       def server_middleware
-        @server_chain ||= Middleware::Chain.new
+        @server_chain ||= Middleware::Chain.new(Sidekiq.default_configuration)
         yield @server_chain if block_given?
         @server_chain
       end
-
-      def constantize(str)
-        names = str.split("::")
-        names.shift if names.empty? || names.first.empty?
-
-        names.inject(Object) do |constant, name|
-          constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
-        end
-      end
     end
   end
 
@@ -83,7 +74,7 @@ module Sidekiq
         true
       elsif Sidekiq::Testing.inline?
         payloads.each do |job|
-          klass = Sidekiq::Testing.constantize(job["class"])
+          klass = Object.const_get(job["class"])
           job["id"] ||= SecureRandom.hex(12)
           job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
           klass.process_job(job_hash)
@@ -218,25 +209,9 @@ module Sidekiq
     #   assert_equal 1, HardJob.jobs.size
     #   assert_equal :something, HardJob.jobs[0]['args'][0]
     #
-    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   MyMailer.delay.send_welcome_email('foo@example.com')
-    #   assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #
     # You can also clear and drain all job types:
     #
-    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
-    #
-    #   MyMailer.delay.send_welcome_email('foo@example.com')
-    #   MyModel.delay.do_something_hard
-    #
-    #   assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   assert_equal 1, Sidekiq::Extensions::DelayedModel.jobs.size
-    #
-    #   Sidekiq::Worker.clear_all # or .drain_all
-    #
-    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
+    #   Sidekiq::Job.clear_all # or .drain_all
     #
     # This can be useful to make sure jobs don't linger between tests:
     #
@@ -318,7 +293,7 @@ module Sidekiq
           job_classes = jobs.map { |job| job["class"] }.uniq
 
           job_classes.each do |job_class|
-            Sidekiq::Testing.constantize(job_class).drain
+            Object.const_get(job_class).drain
           end
         end
       end
@@ -329,13 +304,10 @@ module Sidekiq
     def jobs_for(klass)
       jobs.select do |job|
         marshalled = job["args"][0]
-        marshalled.index(klass.to_s) && YAML.load(marshalled)[0] == klass
+        marshalled.index(klass.to_s) && YAML.safe_load(marshalled)[0] == klass
       end
     end
   end
-
-  Sidekiq::Extensions::DelayedMailer.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedMailer)
-  Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
 end
 
 if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING

data/lib/sidekiq/transaction_aware_client.rb

@@ -5,8 +5,8 @@ require "sidekiq/client"
 
 module Sidekiq
   class TransactionAwareClient
-    def initialize(redis_pool)
-      @redis_client = Client.new(redis_pool)
+    def initialize(pool: nil, config: nil)
+      @redis_client = Client.new(pool: pool, config: config)
     end
 
     def push(item)
@@ -34,11 +34,10 @@ module Sidekiq
     begin
       require "after_commit_everywhere"
     rescue LoadError
-      Sidekiq.logger.error("You need to add after_commit_everywhere to your Gemfile to use Sidekiq's transactional client")
-      raise
+      raise %q(You need to add `gem "after_commit_everywhere"` to your Gemfile to use Sidekiq's transactional client)
     end
 
-    default_job_options["client_class"] = Sidekiq::TransactionAwareClient
+    Sidekiq.default_job_options["client_class"] = Sidekiq::TransactionAwareClient
     Sidekiq::JobUtil::TRANSIENT_ATTRIBUTES << "client_class"
     true
   end

data/lib/sidekiq/version.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.5.12"
+  VERSION = "7.0.6"
+  MAJOR = 7
 end