sidekiq 6.5.6 → 7.0.0.beta1

data/lib/sidekiq/redis_client_adapter.rb

@@ -1,9 +1,7 @@
 # frozen_string_literal: true
 
-require "connection_pool"
 require "redis_client"
 require "redis_client/decorator"
-require "uri"
 
 module Sidekiq
   class RedisClientAdapter
@@ -19,29 +17,10 @@ module Sidekiq
         @client.call("EVALSHA", sha, keys.size, *keys, *argv)
       end
 
-      def brpoplpush(*args)
-        @client.blocking_call(false, "BRPOPLPUSH", *args)
-      end
-
-      def brpop(*args)
-        @client.blocking_call(false, "BRPOP", *args)
-      end
-
-      def set(*args)
-        @client.call("SET", *args) { |r| r == "OK" }
-      end
-      ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
-
-      def sismember(*args)
-        @client.call("SISMEMBER", *args) { |c| c > 0 }
-      end
-
-      def exists?(key)
-        @client.call("EXISTS", key) { |c| c > 0 }
-      end
-
       private
 
+      # this allows us to use methods like `conn.hmset(...)` instead of having to use
+      # redis-client's native `conn.call("hmset", ...)`
       def method_missing(*args, &block)
         @client.call(*args, *block)
       end
@@ -55,27 +34,15 @@ module Sidekiq
     CompatClient = RedisClient::Decorator.create(CompatMethods)
 
     class CompatClient
-      %i[scan sscan zscan hscan].each do |method|
-        alias_method :"#{method}_each", method
-        undef_method method
-      end
-
-      def disconnect!
-        @client.close
-      end
-
-      def connection
-        {id: @client.id}
-      end
-
-      def redis
-        self
-      end
-
+      # underscore methods are not official API
       def _client
         @client
       end
 
+      def _config
+        @client.config
+      end
+
       def message
         yield nil, @queue.pop
       end
@@ -118,9 +85,8 @@ module Sidekiq
       opts = options.dup
 
       if opts[:namespace]
-        Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
-          "Either use the redis adapter or remove the namespace.")
-        Kernel.exit(-127)
+        raise ArgumentError, "Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
+          "Either use the redis adapter or remove the namespace."
       end
 
       opts.delete(:size)
@@ -150,5 +116,3 @@ module Sidekiq
     end
   end
 end
-
-Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter

data/lib/sidekiq/redis_connection.rb

@@ -1,128 +1,32 @@
 # frozen_string_literal: true
 
 require "connection_pool"
-require "redis"
 require "uri"
+require "sidekiq/redis_client_adapter"
 
 module Sidekiq
   module RedisConnection
-    class RedisAdapter
-      BaseError = Redis::BaseError
-      CommandError = Redis::CommandError
-
-      def initialize(options)
-        warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
-        @options = options
-      end
-
-      def new_client
-        namespace = @options[:namespace]
-
-        client = Redis.new client_opts(@options)
-        if namespace
-          begin
-            require "redis/namespace"
-            Redis::Namespace.new(namespace, redis: client)
-          rescue LoadError
-            Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
-                                 "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
-            exit(-127)
-          end
-        else
-          client
-        end
-      end
-
-      private
-
-      def client_opts(options)
-        opts = options.dup
-        if opts[:namespace]
-          opts.delete(:namespace)
-        end
-
-        if opts[:network_timeout]
-          opts[:timeout] = opts[:network_timeout]
-          opts.delete(:network_timeout)
-        end
-
-        # Issue #3303, redis-rb will silently retry an operation.
-        # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
-        # is performed twice but I believe this is much, much rarer
-        # than the reconnect silently fixing a problem; we keep it
-        # on by default.
-        opts[:reconnect_attempts] ||= 1
-
-        opts
-      end
-    end
-
-    @adapter = RedisAdapter
-
     class << self
-      attr_reader :adapter
-
-      # RedisConnection.adapter = :redis
-      # RedisConnection.adapter = :redis_client
-      def adapter=(adapter)
-        raise "no" if adapter == self
-        result = case adapter
-        when :redis
-          RedisAdapter
-        when Class
-          adapter
-        else
-          require "sidekiq/#{adapter}_adapter"
-          nil
-        end
-        @adapter = result if result
-      end
-
       def create(options = {})
         symbolized_options = options.transform_keys(&:to_sym)
+        symbolized_options[:url] ||= determine_redis_provider
 
-        if !symbolized_options[:url] && (u = determine_redis_provider)
-          symbolized_options[:url] = u
-        end
-
-        size = if symbolized_options[:size]
-          symbolized_options[:size]
-        elsif Sidekiq.server?
-          # Give ourselves plenty of connections.  pool is lazy
-          # so we won't create them until we need them.
-          Sidekiq[:concurrency] + 5
-        elsif ENV["RAILS_MAX_THREADS"]
-          Integer(ENV["RAILS_MAX_THREADS"])
-        else
-          5
-        end
-
-        verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
+        logger = symbolized_options.delete(:logger)
+        logger&.info { "Sidekiq #{Sidekiq::VERSION} connecting to Redis with options #{scrub(symbolized_options)}" }
 
-        pool_timeout = symbolized_options[:pool_timeout] || 1
-        log_info(symbolized_options)
+        size = symbolized_options.delete(:size) || 5
+        pool_timeout = symbolized_options.delete(:pool_timeout) || 1
+        pool_name = symbolized_options.delete(:pool_name)
 
-        redis_config = adapter.new(symbolized_options)
-        ConnectionPool.new(timeout: pool_timeout, size: size) do
+        redis_config = Sidekiq::RedisClientAdapter.new(symbolized_options)
+        ConnectionPool.new(timeout: pool_timeout, size: size, name: pool_name) do
           redis_config.new_client
         end
       end
 
       private
 
-      # Sidekiq needs many concurrent Redis connections.
-      #
-      # We need a connection for each Processor.
-      # We need a connection for Pro's real-time change listener
-      # We need a connection to various features to call Redis every few seconds:
-      #   - the process heartbeat.
-      #   - enterprise's leader election
-      #   - enterprise's cron support
-      def verify_sizing(size, concurrency)
-        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
-      end
-
-      def log_info(options)
+      def scrub(options)
         redacted = "REDACTED"
 
         # Deep clone so we can muck with these options all we want and exclude
@@ -140,11 +44,7 @@ module Sidekiq
         scrubbed_options[:sentinels]&.each do |sentinel|
           sentinel[:password] = redacted if sentinel[:password]
         end
-        if Sidekiq.server?
-          Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
-        else
-          Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
-        end
+        scrubbed_options
       end
 
       def determine_redis_provider

data/lib/sidekiq/scheduled.rb

@@ -8,6 +8,8 @@ module Sidekiq
     SETS = %w[retry schedule]
 
     class Enq
+      include Sidekiq::Component
+
       LUA_ZPOPBYSCORE = <<~LUA
         local key, now = KEYS[1], ARGV[1]
         local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
@@ -17,7 +19,9 @@ module Sidekiq
         end
       LUA
 
-      def initialize
+      def initialize(container)
+        @config = container
+        @client = Sidekiq::Client.new(config: container)
         @done = false
         @lua_zpopbyscore_sha = nil
       end
@@ -25,15 +29,15 @@ module Sidekiq
       def enqueue_jobs(sorted_sets = SETS)
         # A job's "score" in Redis is the time at which it should be processed.
         # Just check Redis for the set of jobs with a timestamp before now.
-        Sidekiq.redis do |conn|
+        redis do |conn|
           sorted_sets.each do |sorted_set|
             # Get next item in the queue with score (time to execute) <= now.
             # We need to go through the list one at a time to reduce the risk of something
             # going wrong between the time jobs are popped from the scheduled queue and when
             # they are pushed onto a work queue and losing the jobs.
             while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s]))
-              Sidekiq::Client.push(Sidekiq.load_json(job))
-              Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
+              @client.push(Sidekiq.load_json(job))
+              logger.debug { "enqueued #{sorted_set}: #{job}" }
             end
           end
         end
@@ -47,12 +51,11 @@ module Sidekiq
 
       def zpopbyscore(conn, keys: nil, argv: nil)
         if @lua_zpopbyscore_sha.nil?
-          raw_conn = conn.respond_to?(:redis) ? conn.redis : conn
-          @lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
+          @lua_zpopbyscore_sha = conn.script(:load, LUA_ZPOPBYSCORE)
         end
 
         conn.evalsha(@lua_zpopbyscore_sha, keys, argv)
-      rescue RedisConnection.adapter::CommandError => e
+      rescue RedisClient::CommandError => e
         raise unless e.message.start_with?("NOSCRIPT")
 
         @lua_zpopbyscore_sha = nil
@@ -70,9 +73,9 @@ module Sidekiq
 
       INITIAL_WAIT = 10
 
-      def initialize(options)
-        @config = options
-        @enq = (options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
+      def initialize(config)
+        @config = config
+        @enq = (config[:scheduled_enq] || Sidekiq::Scheduled::Enq).new(config)
         @sleeper = ConnectionPool::TimedStack.new
         @done = false
         @thread = nil
@@ -82,14 +85,10 @@ module Sidekiq
       # Shut down this instance, will pause until the thread is dead.
       def terminate
         @done = true
-        @enq.terminate if @enq.respond_to?(:terminate)
+        @enq.terminate
 
-        if @thread
-          t = @thread
-          @thread = nil
-          @sleeper << 0
-          t.value
-        end
+        @sleeper << 0
+        @thread&.value
       end
 
       def start
@@ -147,13 +146,16 @@ module Sidekiq
         # As we run more processes, the scheduling interval average will approach an even spread
         # between 0 and poll interval so we don't need this artifical boost.
         #
-        if process_count < 10
+        count = process_count
+        interval = poll_interval_average(count)
+
+        if count < 10
           # For small clusters, calculate a random interval that is ±50% the desired average.
-          poll_interval_average * rand + poll_interval_average.to_f / 2
+          interval * rand + interval.to_f / 2
         else
           # With 10+ processes, we should have enough randomness to get decent polling
           # across the entire timespan
-          poll_interval_average * rand
+          interval * rand
         end
       end
 
@@ -170,14 +172,14 @@ module Sidekiq
       # the same time: the thundering herd problem.
       #
       # We only do this if poll_interval_average is unset (the default).
-      def poll_interval_average
-        @config[:poll_interval_average] ||= scaled_poll_interval
+      def poll_interval_average(count)
+        @config[:poll_interval_average] || scaled_poll_interval(count)
       end
 
       # Calculates an average poll interval based on the number of known Sidekiq processes.
       # This minimizes a single point of failure by dispersing check-ins but without taxing
       # Redis if you run many Sidekiq processes.
-      def scaled_poll_interval
+      def scaled_poll_interval(process_count)
         process_count * @config[:average_scheduled_poll_interval]
       end
 
@@ -187,9 +189,35 @@ module Sidekiq
         pcount
       end
 
+      # A copy of Sidekiq::ProcessSet#cleanup because server
+      # should never depend on sidekiq/api.
+      def cleanup
+        # dont run cleanup more than once per minute
+        return 0 unless redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
+
+        count = 0
+        redis do |conn|
+          procs = conn.sscan("processes").to_a
+          heartbeats = conn.pipelined { |pipeline|
+            procs.each do |key|
+              pipeline.hget(key, "info")
+            end
+          }
+
+          # the hash named key has an expiry of 60 seconds.
+          # if it's not found, that means the process has not reported
+          # in to Redis and probably died.
+          to_prune = procs.select.with_index { |proc, i|
+            heartbeats[i].nil?
+          }
+          count = conn.srem("processes", to_prune) unless to_prune.empty?
+        end
+        count
+      end
+
       def initial_wait
-        # Have all processes sleep between 5-15 seconds.  10 seconds
-        # to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
+        # Have all processes sleep between 5-15 seconds. 10 seconds to give time for
+        # the heartbeat to register (if the poll interval is going to be calculated by the number
         # of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
         total = 0
         total += INITIAL_WAIT unless @config[:poll_interval_average]
@@ -197,6 +225,11 @@ module Sidekiq
 
         @sleeper.pop(total)
       rescue Timeout::Error
+      ensure
+        # periodically clean out the `processes` set in Redis which can collect
+        # references to dead processes over time. The process count affects how
+        # often we scan for scheduled jobs.
+        cleanup
       end
     end
   end

data/lib/sidekiq/testing.rb

@@ -51,19 +51,10 @@ module Sidekiq
       end
 
       def server_middleware
-        @server_chain ||= Middleware::Chain.new
+        @server_chain ||= Middleware::Chain.new(Sidekiq.default_configuration)
         yield @server_chain if block_given?
         @server_chain
       end
-
-      def constantize(str)
-        names = str.split("::")
-        names.shift if names.empty? || names.first.empty?
-
-        names.inject(Object) do |constant, name|
-          constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
-        end
-      end
     end
   end
 
@@ -83,7 +74,7 @@ module Sidekiq
         true
       elsif Sidekiq::Testing.inline?
         payloads.each do |job|
-          klass = Sidekiq::Testing.constantize(job["class"])
+          klass = Object.const_get(job["class"])
           job["id"] ||= SecureRandom.hex(12)
           job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
           klass.process_job(job_hash)
@@ -218,26 +209,10 @@ module Sidekiq
     #   assert_equal 1, HardJob.jobs.size
     #   assert_equal :something, HardJob.jobs[0]['args'][0]
     #
-    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   MyMailer.delay.send_welcome_email('foo@example.com')
-    #   assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #
     # You can also clear and drain all job types:
     #
-    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
-    #
-    #   MyMailer.delay.send_welcome_email('foo@example.com')
-    #   MyModel.delay.do_something_hard
-    #
-    #   assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   assert_equal 1, Sidekiq::Extensions::DelayedModel.jobs.size
-    #
     #   Sidekiq::Worker.clear_all # or .drain_all
     #
-    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
-    #
     # This can be useful to make sure jobs don't linger between tests:
     #
     #   RSpec.configure do |config|
@@ -318,7 +293,7 @@ module Sidekiq
           job_classes = jobs.map { |job| job["class"] }.uniq
 
           job_classes.each do |job_class|
-            Sidekiq::Testing.constantize(job_class).drain
+            Object.const_get(job_class).drain
           end
         end
       end
@@ -329,13 +304,10 @@ module Sidekiq
     def jobs_for(klass)
       jobs.select do |job|
         marshalled = job["args"][0]
-        marshalled.index(klass.to_s) && YAML.load(marshalled)[0] == klass
+        marshalled.index(klass.to_s) && YAML.safe_load(marshalled)[0] == klass
       end
     end
   end
-
-  Sidekiq::Extensions::DelayedMailer.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedMailer)
-  Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
 end
 
 if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING

data/lib/sidekiq/transaction_aware_client.rb

@@ -5,8 +5,8 @@ require "sidekiq/client"
 
 module Sidekiq
   class TransactionAwareClient
-    def initialize(redis_pool)
-      @redis_client = Client.new(redis_pool)
+    def initialize(pool: nil, config: nil)
+      @redis_client = Client.new(pool: pool, config: config)
     end
 
     def push(item)
@@ -34,11 +34,10 @@ module Sidekiq
     begin
       require "after_commit_everywhere"
     rescue LoadError
-      Sidekiq.logger.error("You need to add after_commit_everywhere to your Gemfile to use Sidekiq's transactional client")
-      raise
+      raise %q(You need to add `gem "after_commit_everywhere"` to your Gemfile to use Sidekiq's transactional client)
     end
 
-    default_job_options["client_class"] = Sidekiq::TransactionAwareClient
+    Sidekiq.default_job_options["client_class"] = Sidekiq::TransactionAwareClient
     Sidekiq::JobUtil::TRANSIENT_ATTRIBUTES << "client_class"
     true
   end

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.5.6"
+  VERSION = "7.0.0.beta1"
 end

data/lib/sidekiq/web/application.rb

@@ -307,7 +307,7 @@ module Sidekiq
     end
 
     get "/stats/queues" do
-      json Sidekiq::Stats::Queues.new.lengths
+      json Sidekiq::Stats.new.queues
     end
 
     def call(env)

data/lib/sidekiq/web/csrf_protection.rb

@@ -152,7 +152,7 @@ module Sidekiq
         # value and decrypt it
         token_length = masked_token.length / 2
         one_time_pad = masked_token[0...token_length]
-        encrypted_token = masked_token[token_length..-1]
+        encrypted_token = masked_token[token_length..]
         xor_byte_strings(one_time_pad, encrypted_token)
       end
 

data/lib/sidekiq/web/helpers.rb

@@ -155,7 +155,7 @@ module Sidekiq
       @sorted_processes ||= begin
         return processes unless processes.all? { |p| p["hostname"] }
 
-        split_characters = /[._-]/
+        split_characters = /[._-]+/
 
         padding = processes.flat_map { |p| p["hostname"].split(split_characters) }.map(&:size).max
 
@@ -175,18 +175,14 @@ module Sidekiq
       @stats ||= Sidekiq::Stats.new
     end
 
-    def redis_connection
+    def redis_url
       Sidekiq.redis do |conn|
-        conn.connection[:id]
+        conn._config.server_url
       end
     end
 
-    def namespace
-      @ns ||= Sidekiq.redis { |conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
-    end
-
     def redis_info
-      Sidekiq.redis_info
+      Sidekiq.default_configuration.redis_info
     end
 
     def root_path
@@ -324,7 +320,7 @@ module Sidekiq
     end
 
     def environment_title_prefix
-      environment = Sidekiq[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      environment = Sidekiq.default_configuration[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
 
       "[#{environment.upcase}] " unless environment == "production"
     end
@@ -337,13 +333,6 @@ module Sidekiq
       Time.now.utc.strftime("%H:%M:%S UTC")
     end
 
-    def redis_connection_and_namespace
-      @redis_connection_and_namespace ||= begin
-        namespace_suffix = namespace.nil? ? "" : "##{namespace}"
-        "#{redis_connection}#{namespace_suffix}"
-      end
-    end
-
     def retry_or_delete_or_kill(job, params)
       if params["retry"]
         job.retry

data/lib/sidekiq/web.rb

@@ -30,13 +30,10 @@ module Sidekiq
       "Queues" => "queues",
       "Retries" => "retries",
       "Scheduled" => "scheduled",
-      "Dead" => "morgue"
+      "Dead" => "morgue",
+      "Metrics" => "metrics"
     }
 
-    if ENV["SIDEKIQ_METRICS_BETA"] == "1"
-      DEFAULT_TABS["Metrics"] = "metrics"
-    end
-
     class << self
       def settings
         self
@@ -79,14 +76,6 @@ module Sidekiq
         send(:"#{attribute}=", value)
       end
 
-      def sessions=(val)
-        puts "WARNING: Sidekiq::Web.sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
-      end
-
-      def session_secret=(val)
-        puts "WARNING: Sidekiq::Web.session_secret= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
-      end
-
       attr_accessor :app_url, :redis_pool
       attr_writer :locales, :views
     end
@@ -133,10 +122,6 @@ module Sidekiq
       send(:"#{attribute}=", value)
     end
 
-    def sessions=(val)
-      puts "Sidekiq::Web#sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller[2..2].first}"
-    end
-
     def self.register(extension)
       extension.registered(WebApplication)
     end

data/lib/sidekiq/worker_compatibility_alias.rb

@@ -0,0 +1,13 @@
+require "sidekiq/job"
+
+module Sidekiq
+  # Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
+  # Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
+  #
+  # The term "worker" is too generic and overly confusing, used in several
+  # different contexts meaning different things. Many people call a Sidekiq
+  # process a "worker". Some people call the thread that executes jobs a
+  # "worker". This change brings Sidekiq closer to ActiveJob where your job
+  # classes extend ApplicationJob.
+  Worker = Job
+end