sidekiq 6.4.2 → 6.5.0

data/lib/sidekiq/launcher.rb

@@ -3,11 +3,12 @@
 require "sidekiq/manager"
 require "sidekiq/fetch"
 require "sidekiq/scheduled"
+require "sidekiq/ring_buffer"
 
 module Sidekiq
   # The Launcher starts the Manager and Poller threads and provides the process heartbeat.
   class Launcher
-    include Util
+    include Sidekiq::Component
 
     STATS_TTL = 5 * 365 * 24 * 60 * 60 # 5 years
 
@@ -22,11 +23,11 @@ module Sidekiq
     attr_accessor :manager, :poller, :fetcher
 
     def initialize(options)
+      @config = options
       options[:fetch] ||= BasicFetch.new(options)
       @manager = Sidekiq::Manager.new(options)
-      @poller = Sidekiq::Scheduled::Poller.new
+      @poller = Sidekiq::Scheduled::Poller.new(options)
       @done = false
-      @options = options
     end
 
     def run
@@ -45,7 +46,7 @@ module Sidekiq
 
     # Shuts down this Sidekiq instance. Waits up to the deadline for all jobs to complete.
     def stop
-      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @options[:timeout]
+      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @config[:timeout]
 
       @done = true
       @manager.quiet
@@ -55,8 +56,8 @@ module Sidekiq
 
       # Requeue everything in case there was a thread which fetched a job while the process was stopped.
       # This call is a no-op in Sidekiq but necessary for Sidekiq Pro.
-      strategy = @options[:fetch]
-      strategy.bulk_requeue([], @options)
+      strategy = @config[:fetch]
+      strategy.bulk_requeue([], @config)
 
       clear_heartbeat
     end
@@ -74,14 +75,14 @@ module Sidekiq
         heartbeat
         sleep BEAT_PAUSE
       end
-      Sidekiq.logger.info("Heartbeat stopping...")
+      logger.info("Heartbeat stopping...")
     end
 
     def clear_heartbeat
       # Remove record from Redis since we are shutting down.
       # Note we don't stop the heartbeat thread; if the process
       # doesn't actually exit, it'll reappear in the Web UI.
-      Sidekiq.redis do |conn|
+      redis do |conn|
         conn.pipelined do |pipeline|
           pipeline.srem("processes", identity)
           pipeline.unlink("#{identity}:work")
@@ -134,7 +135,7 @@ module Sidekiq
 
         nowdate = Time.now.utc.strftime("%Y-%m-%d")
 
-        Sidekiq.redis do |conn|
+        redis do |conn|
           conn.multi do |transaction|
             transaction.incrby("stat:processed", procd)
             transaction.incrby("stat:processed:#{nowdate}", procd)
@@ -161,7 +162,7 @@ module Sidekiq
         fails = procd = 0
         kb = memory_usage(::Process.pid)
 
-        _, exists, _, _, msg = Sidekiq.redis { |conn|
+        _, exists, _, _, msg = redis { |conn|
           conn.multi { |transaction|
             transaction.sadd("processes", key)
             transaction.exists?(key)
@@ -169,7 +170,7 @@ module Sidekiq
               "busy", curstate.size,
               "beat", Time.now.to_f,
               "rtt_us", rtt,
-              "quiet", @done,
+              "quiet", @done.to_s,
               "rss", kb)
             transaction.expire(key, 60)
             transaction.rpop("#{key}-signals")
@@ -199,7 +200,7 @@ module Sidekiq
 
     def check_rtt
       a = b = 0
-      Sidekiq.redis do |x|
+      redis do |x|
         a = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
         x.ping
         b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
@@ -210,7 +211,7 @@ module Sidekiq
       # Workable is < 10,000µs
       # Log a warning if it's a disaster.
       if RTT_READINGS.all? { |x| x > RTT_WARNING_LEVEL }
-        Sidekiq.logger.warn <<~EOM
+        logger.warn <<~EOM
           Your Redis network connection is performing extremely poorly.
           Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
           Ensure Redis is running in the same AZ or datacenter as Sidekiq.
@@ -247,10 +248,10 @@ module Sidekiq
         "hostname" => hostname,
         "started_at" => Time.now.to_f,
         "pid" => ::Process.pid,
-        "tag" => @options[:tag] || "",
-        "concurrency" => @options[:concurrency],
-        "queues" => @options[:queues].uniq,
-        "labels" => @options[:labels],
+        "tag" => @config[:tag] || "",
+        "concurrency" => @config[:concurrency],
+        "queues" => @config[:queues].uniq,
+        "labels" => @config[:labels],
         "identity" => identity
       }
     end

data/lib/sidekiq/logger.rb

@@ -18,7 +18,7 @@ module Sidekiq
     end
 
     def self.add(k, v)
-      Thread.current[:sidekiq_context][k] = v
+      current[k] = v
     end
   end
 

data/lib/sidekiq/manager.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "sidekiq/util"
 require "sidekiq/processor"
 require "sidekiq/fetch"
 require "set"
@@ -21,29 +20,26 @@ module Sidekiq
   # the shutdown process.  The other tasks are performed by other threads.
   #
   class Manager
-    include Util
+    include Sidekiq::Component
 
     attr_reader :workers
-    attr_reader :options
 
     def initialize(options = {})
+      @config = options
       logger.debug { options.inspect }
-      @options = options
       @count = options[:concurrency] || 10
       raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
 
       @done = false
       @workers = Set.new
       @count.times do
-        @workers << Processor.new(self, options)
+        @workers << Processor.new(@config, &method(:processor_result))
       end
       @plock = Mutex.new
     end
 
     def start
-      @workers.each do |x|
-        x.start
-      end
+      @workers.each(&:start)
     end
 
     def quiet
@@ -51,7 +47,7 @@ module Sidekiq
       @done = true
 
       logger.info { "Terminating quiet threads" }
-      @workers.each { |x| x.terminate }
+      @workers.each(&:terminate)
       fire_event(:quiet, reverse: true)
     end
 
@@ -72,17 +68,11 @@ module Sidekiq
       hard_shutdown
     end
 
-    def processor_stopped(processor)
-      @plock.synchronize do
-        @workers.delete(processor)
-      end
-    end
-
-    def processor_died(processor, reason)
+    def processor_result(processor, reason = nil)
       @plock.synchronize do
         @workers.delete(processor)
         unless @done
-          p = Processor.new(self, options)
+          p = Processor.new(@config, &method(:processor_result))
           @workers << p
           p.start
         end
@@ -107,7 +97,7 @@ module Sidekiq
         jobs = cleanup.map { |p| p.job }.compact
 
         logger.warn { "Terminating #{cleanup.size} busy threads" }
-        logger.warn { "Jobs still in progress #{jobs.inspect}" }
+        logger.debug { "Jobs still in progress #{jobs.inspect}" }
 
         # Re-enqueue unfinished jobs
         # NOTE: You may notice that we may push a job back to redis before
@@ -115,8 +105,8 @@ module Sidekiq
         # contract says that jobs are run AT LEAST once. Process termination
         # is delayed until we're certain the jobs are back in Redis because
         # it is worse to lose a job than to run it twice.
-        strategy = @options[:fetch]
-        strategy.bulk_requeue(jobs, @options)
+        strategy = @config[:fetch]
+        strategy.bulk_requeue(jobs, @config)
       end
 
       cleanup.each do |processor|
@@ -129,5 +119,18 @@ module Sidekiq
       deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + 3
       wait_for(deadline) { @workers.empty? }
     end
+
+    # hack for quicker development / testing environment #2774
+    PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
+
+    # Wait for the orblock to be true or the deadline passed.
+    def wait_for(deadline, &condblock)
+      remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      while remaining > PAUSE_TIME
+        return if condblock.call
+        sleep PAUSE_TIME
+        remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
   end
 end

data/lib/sidekiq/middleware/chain.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "sidekiq/middleware/modules"
+
 module Sidekiq
   # Middleware is code configured to run before/after
   # a message is processed.  It is patterned after Rack
@@ -44,10 +46,12 @@ module Sidekiq
   # This is an example of a minimal server middleware:
   #
   # class MyServerHook
+  #   include Sidekiq::ServerMiddleware
   #   def call(job_instance, msg, queue)
-  #     puts "Before job"
+  #     logger.info "Before job"
+  #     redis {|conn| conn.get("foo") } # do something in Redis
   #     yield
-  #     puts "After job"
+  #     logger.info "After job"
   #   end
   # end
   #
@@ -56,10 +60,11 @@ module Sidekiq
   # to Redis:
   #
   # class MyClientHook
+  #   include Sidekiq::ClientMiddleware
   #   def call(job_class, msg, queue, redis_pool)
-  #     puts "Before push"
+  #     logger.info "Before push"
   #     result = yield
-  #     puts "After push"
+  #     logger.info "After push"
   #     result
   #   end
   # end
@@ -76,7 +81,8 @@ module Sidekiq
         entries.each(&block)
       end
 
-      def initialize
+      def initialize(config = nil)
+        @config = config
         @entries = nil
         yield self if block_given?
       end
@@ -91,24 +97,24 @@ module Sidekiq
 
       def add(klass, *args)
         remove(klass)
-        entries << Entry.new(klass, *args)
+        entries << Entry.new(@config, klass, *args)
       end
 
       def prepend(klass, *args)
         remove(klass)
-        entries.insert(0, Entry.new(klass, *args))
+        entries.insert(0, Entry.new(@config, klass, *args))
       end
 
       def insert_before(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
-        new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
+        new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i)
         i = entries.index { |entry| entry.klass == oldklass } || 0
         entries.insert(i, new_entry)
       end
 
       def insert_after(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
-        new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
+        new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i)
         i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
         entries.insert(i + 1, new_entry)
       end
@@ -149,13 +155,16 @@ module Sidekiq
     class Entry
       attr_reader :klass
 
-      def initialize(klass, *args)
+      def initialize(config, klass, *args)
+        @config = config
         @klass = klass
         @args = args
       end
 
       def make_new
-        @klass.new(*@args)
+        x = @klass.new(*@args)
+        x.config = @config if @config && x.respond_to?(:config=)
+        x
       end
     end
   end

data/lib/sidekiq/middleware/current_attributes.rb

@@ -15,6 +15,8 @@ module Sidekiq
   #
   module CurrentAttributes
     class Save
+      include Sidekiq::ClientMiddleware
+
       def initialize(cattr)
         @klass = cattr
       end
@@ -31,6 +33,8 @@ module Sidekiq
     end
 
     class Load
+      include Sidekiq::ServerMiddleware
+
       def initialize(cattr)
         @klass = cattr
       end

data/lib/sidekiq/middleware/i18n.rb

@@ -10,6 +10,7 @@ module Sidekiq::Middleware::I18n
   # Get the current locale and store it in the message
   # to be sent to Sidekiq.
   class Client
+    include Sidekiq::ClientMiddleware
     def call(_jobclass, job, _queue, _redis)
       job["locale"] ||= I18n.locale
       yield
@@ -18,6 +19,7 @@ module Sidekiq::Middleware::I18n
 
   # Pull the msg locale out and set the current thread to use it.
   class Server
+    include Sidekiq::ServerMiddleware
     def call(_jobclass, job, _queue, &block)
       I18n.with_locale(job.fetch("locale", I18n.default_locale), &block)
     end

data/lib/sidekiq/middleware/modules.rb

@@ -0,0 +1,19 @@
+module Sidekiq
+  module ServerMiddleware
+    attr_accessor :config
+    def redis_pool
+      config.redis_pool
+    end
+
+    def logger
+      config.logger
+    end
+
+    def redis(&block)
+      config.redis(&block)
+    end
+  end
+
+  # no difference for now
+  ClientMiddleware = ServerMiddleware
+end

data/lib/sidekiq/paginator.rb

@@ -19,9 +19,9 @@ module Sidekiq
           total_size, items = conn.multi { |transaction|
             transaction.zcard(key)
             if rev
-              transaction.zrevrange(key, starting, ending, with_scores: true)
+              transaction.zrevrange(key, starting, ending, withscores: true)
             else
-              transaction.zrange(key, starting, ending, with_scores: true)
+              transaction.zrange(key, starting, ending, withscores: true)
             end
           }
           [current_page, total_size, items]

data/lib/sidekiq/processor.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "sidekiq/util"
 require "sidekiq/fetch"
 require "sidekiq/job_logger"
 require "sidekiq/job_retry"
@@ -15,29 +14,30 @@ module Sidekiq
   #   b. run the middleware chain
   #   c. call #perform
   #
-  # A Processor can exit due to shutdown (processor_stopped)
-  # or due to an error during job execution (processor_died)
+  # A Processor can exit due to shutdown or due to
+  # an error during job execution.
   #
   # If an error occurs in the job execution, the
   # Processor calls the Manager to create a new one
   # to replace itself and exits.
   #
   class Processor
-    include Util
+    include Sidekiq::Component
 
     attr_reader :thread
     attr_reader :job
 
-    def initialize(mgr, options)
-      @mgr = mgr
+    def initialize(options, &block)
+      @callback = block
       @down = false
       @done = false
       @job = nil
       @thread = nil
+      @config = options
       @strategy = options[:fetch]
       @reloader = options[:reloader] || proc { |&block| block.call }
       @job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
-      @retrier = Sidekiq::JobRetry.new
+      @retrier = Sidekiq::JobRetry.new(options)
     end
 
     def terminate(wait = false)
@@ -66,14 +66,14 @@ module Sidekiq
 
     def run
       process_one until @done
-      @mgr.processor_stopped(self)
+      @callback.call(self)
     rescue Sidekiq::Shutdown
-      @mgr.processor_stopped(self)
+      @callback.call(self)
     rescue Exception => ex
-      @mgr.processor_died(self, ex)
+      @callback.call(self, ex)
     end
 
-    def process_one
+    def process_one(&block)
       @job = fetch
       process(@job) if @job
       @job = nil
@@ -160,7 +160,7 @@ module Sidekiq
       ack = false
       begin
         dispatch(job_hash, queue, jobstr) do |inst|
-          Sidekiq.server_middleware.invoke(inst, job_hash, queue) do
+          @config.server_middleware.invoke(inst, job_hash, queue) do
             execute_job(inst, job_hash["args"])
           end
         end

data/lib/sidekiq/rails.rb

@@ -38,12 +38,12 @@ module Sidekiq
     end
 
     initializer "sidekiq.rails_logger" do
-      Sidekiq.configure_server do |_|
+      Sidekiq.configure_server do |config|
         # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
         # it will appear in the Sidekiq console with all of the job context. See #5021 and
         # https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
-        unless ::Rails.logger == ::Sidekiq.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
-          ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(::Sidekiq.logger))
+        unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
+          ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
         end
       end
     end
@@ -60,8 +60,8 @@ module Sidekiq
     #
     # None of this matters on the client-side, only within the Sidekiq process itself.
     config.after_initialize do
-      Sidekiq.configure_server do |_|
-        Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
+      Sidekiq.configure_server do |config|
+        config[:reloader] = Sidekiq::Rails::Reloader.new
       end
     end
   end

data/lib/sidekiq/redis_client_adapter.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require "connection_pool"
+require "redis_client"
+require "redis_client/decorator"
+require "uri"
+
+module Sidekiq
+  class RedisClientAdapter
+    BaseError = RedisClient::Error
+    CommandError = RedisClient::CommandError
+
+    module CompatMethods
+      def info
+        @client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
+      end
+
+      def evalsha(sha, keys, argv)
+        @client.call("EVALSHA", sha, keys.size, *keys, *argv)
+      end
+
+      def brpoplpush(*args)
+        @client.blocking_call(false, "BRPOPLPUSH", *args)
+      end
+
+      def brpop(*args)
+        @client.blocking_call(false, "BRPOP", *args)
+      end
+
+      def set(*args)
+        @client.call("SET", *args) { |r| r == "OK" }
+      end
+      ruby2_keywords :set if respond_to?(:ruby2_keywords, true)
+
+      def sismember(*args)
+        @client.call("SISMEMBER", *args) { |c| c > 0 }
+      end
+
+      def exists?(key)
+        @client.call("EXISTS", key) { |c| c > 0 }
+      end
+
+      private
+
+      def method_missing(*args, &block)
+        @client.call(*args, *block)
+      end
+      ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
+
+      def respond_to_missing?(name, include_private = false)
+        super # Appease the linter. We can't tell what is a valid command.
+      end
+    end
+
+    CompatClient = RedisClient::Decorator.create(CompatMethods)
+
+    class CompatClient
+      %i[scan sscan zscan hscan].each do |method|
+        alias_method :"#{method}_each", method
+        undef_method method
+      end
+
+      def disconnect!
+        @client.close
+      end
+
+      def connection
+        {id: @client.id}
+      end
+
+      def redis
+        self
+      end
+
+      def _client
+        @client
+      end
+
+      def message
+        yield nil, @queue.pop
+      end
+
+      # NB: this method does not return
+      def subscribe(chan)
+        @queue = ::Queue.new
+
+        pubsub = @client.pubsub
+        pubsub.call("subscribe", chan)
+
+        loop do
+          evt = pubsub.next_event
+          next if evt.nil?
+          next unless evt[0] == "message" && evt[1] == chan
+
+          (_, _, msg) = evt
+          @queue << msg
+          yield self
+        end
+      end
+    end
+
+    def initialize(options)
+      opts = client_opts(options)
+      @config = if opts.key?(:sentinels)
+        RedisClient.sentinel(**opts)
+      else
+        RedisClient.config(**opts)
+      end
+    end
+
+    def new_client
+      CompatClient.new(@config.new_client)
+    end
+
+    private
+
+    def client_opts(options)
+      opts = options.dup
+
+      if opts[:namespace]
+        Sidekiq.logger.error("Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
+          "Either use the redis adapter or remove the namespace.")
+        Kernel.exit(-127)
+      end
+
+      opts.delete(:size)
+      opts.delete(:pool_timeout)
+
+      if opts[:network_timeout]
+        opts[:timeout] = opts[:network_timeout]
+        opts.delete(:network_timeout)
+      end
+
+      if opts[:driver]
+        opts[:driver] = opts[:driver].to_sym
+      end
+
+      opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
+      opts[:role] = opts[:role].to_sym if opts.key?(:role)
+      opts.delete(:url) if opts.key?(:sentinels)
+
+      # Issue #3303, redis-rb will silently retry an operation.
+      # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
+      # is performed twice but I believe this is much, much rarer
+      # than the reconnect silently fixing a problem; we keep it
+      # on by default.
+      opts[:reconnect_attempts] ||= 1
+
+      opts
+    end
+  end
+end
+
+Sidekiq::RedisConnection.adapter = Sidekiq::RedisClientAdapter