sidekiq 6.0.7 → 6.4.1

data/lib/sidekiq/launcher.rb

@@ -22,6 +22,7 @@ module Sidekiq
     attr_accessor :manager, :poller, :fetcher
 
     def initialize(options)
+      options[:fetch] ||= BasicFetch.new(options)
       @manager = Sidekiq::Manager.new(options)
       @poller = Sidekiq::Scheduled::Poller.new
       @done = false
@@ -56,7 +57,7 @@ module Sidekiq
 
       # Requeue everything in case there was a worker who grabbed work while stopped
       # This call is a no-op in Sidekiq but necessary for Sidekiq Pro.
-      strategy = (@options[:fetch] || Sidekiq::BasicFetch)
+      strategy = @options[:fetch]
       strategy.bulk_requeue([], @options)
 
       clear_heartbeat
@@ -68,10 +69,12 @@ module Sidekiq
 
     private unless $TESTING
 
+    BEAT_PAUSE = 5
+
     def start_heartbeat
       loop do
         heartbeat
-        sleep 5
+        sleep BEAT_PAUSE
       end
       Sidekiq.logger.info("Heartbeat stopping...")
     end
@@ -81,9 +84,9 @@ module Sidekiq
       # Note we don't stop the heartbeat thread; if the process
       # doesn't actually exit, it'll reappear in the Web UI.
       Sidekiq.redis do |conn|
-        conn.pipelined do
-          conn.srem("processes", identity)
-          conn.unlink("#{identity}:workers")
+        conn.pipelined do |pipeline|
+          pipeline.srem("processes", identity)
+          pipeline.unlink("#{identity}:workers")
         end
       end
     rescue
@@ -104,14 +107,14 @@ module Sidekiq
       nowdate = Time.now.utc.strftime("%Y-%m-%d")
       begin
         Sidekiq.redis do |conn|
-          conn.pipelined do
-            conn.incrby("stat:processed", procd)
-            conn.incrby("stat:processed:#{nowdate}", procd)
-            conn.expire("stat:processed:#{nowdate}", STATS_TTL)
-
-            conn.incrby("stat:failed", fails)
-            conn.incrby("stat:failed:#{nowdate}", fails)
-            conn.expire("stat:failed:#{nowdate}", STATS_TTL)
+          conn.pipelined do |pipeline|
+            pipeline.incrby("stat:processed", procd)
+            pipeline.incrby("stat:processed:#{nowdate}", procd)
+            pipeline.expire("stat:processed:#{nowdate}", STATS_TTL)
+
+            pipeline.incrby("stat:failed", fails)
+            pipeline.incrby("stat:failed:#{nowdate}", fails)
+            pipeline.expire("stat:failed:#{nowdate}", STATS_TTL)
           end
         end
       rescue => ex
@@ -135,32 +138,40 @@ module Sidekiq
         nowdate = Time.now.utc.strftime("%Y-%m-%d")
 
         Sidekiq.redis do |conn|
-          conn.multi do
-            conn.incrby("stat:processed", procd)
-            conn.incrby("stat:processed:#{nowdate}", procd)
-            conn.expire("stat:processed:#{nowdate}", STATS_TTL)
+          conn.multi do |transaction|
+            transaction.incrby("stat:processed", procd)
+            transaction.incrby("stat:processed:#{nowdate}", procd)
+            transaction.expire("stat:processed:#{nowdate}", STATS_TTL)
 
-            conn.incrby("stat:failed", fails)
-            conn.incrby("stat:failed:#{nowdate}", fails)
-            conn.expire("stat:failed:#{nowdate}", STATS_TTL)
+            transaction.incrby("stat:failed", fails)
+            transaction.incrby("stat:failed:#{nowdate}", fails)
+            transaction.expire("stat:failed:#{nowdate}", STATS_TTL)
 
-            conn.unlink(workers_key)
+            transaction.unlink(workers_key)
             curstate.each_pair do |tid, hash|
-              conn.hset(workers_key, tid, Sidekiq.dump_json(hash))
+              transaction.hset(workers_key, tid, Sidekiq.dump_json(hash))
             end
-            conn.expire(workers_key, 60)
+            transaction.expire(workers_key, 60)
           end
         end
 
+        rtt = check_rtt
+
         fails = procd = 0
+        kb = memory_usage(::Process.pid)
 
         _, exists, _, _, msg = Sidekiq.redis { |conn|
-          conn.multi {
-            conn.sadd("processes", key)
-            conn.exists(key)
-            conn.hmset(key, "info", to_json, "busy", curstate.size, "beat", Time.now.to_f, "quiet", @done)
-            conn.expire(key, 60)
-            conn.rpop("#{key}-signals")
+          conn.multi { |transaction|
+            transaction.sadd("processes", key)
+            transaction.exists?(key)
+            transaction.hmset(key, "info", to_json,
+              "busy", curstate.size,
+              "beat", Time.now.to_f,
+              "rtt_us", rtt,
+              "quiet", @done,
+              "rss", kb)
+            transaction.expire(key, 60)
+            transaction.rpop("#{key}-signals")
           }
         }
 
@@ -179,27 +190,74 @@ module Sidekiq
       end
     end
 
-    def to_data
-      @data ||= begin
-        {
-          "hostname" => hostname,
-          "started_at" => Time.now.to_f,
-          "pid" => ::Process.pid,
-          "tag" => @options[:tag] || "",
-          "concurrency" => @options[:concurrency],
-          "queues" => @options[:queues].uniq,
-          "labels" => @options[:labels],
-          "identity" => identity
-        }
+    # We run the heartbeat every five seconds.
+    # Capture five samples of RTT, log a warning if each sample
+    # is above our warning threshold.
+    RTT_READINGS = RingBuffer.new(5)
+    RTT_WARNING_LEVEL = 50_000
+
+    def check_rtt
+      a = b = 0
+      Sidekiq.redis do |x|
+        a = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
+        x.ping
+        b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
       end
+      rtt = b - a
+      RTT_READINGS << rtt
+      # Ideal RTT for Redis is < 1000µs
+      # Workable is < 10,000µs
+      # Log a warning if it's a disaster.
+      if RTT_READINGS.all? { |x| x > RTT_WARNING_LEVEL }
+        Sidekiq.logger.warn <<~EOM
+          Your Redis network connection is performing extremely poorly.
+          Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
+          Ensure Redis is running in the same AZ or datacenter as Sidekiq.
+          If these values are close to 100,000, that means your Sidekiq process may be
+          CPU overloaded; see https://github.com/mperham/sidekiq/discussions/5039
+        EOM
+        RTT_READINGS.reset
+      end
+      rtt
+    end
+
+    MEMORY_GRABBER = case RUBY_PLATFORM
+    when /linux/
+      ->(pid) {
+        IO.readlines("/proc/#{$$}/status").each do |line|
+          next unless line.start_with?("VmRSS:")
+          break line.split[1].to_i
+        end
+      }
+    when /darwin|bsd/
+      ->(pid) {
+        `ps -o pid,rss -p #{pid}`.lines.last.split.last.to_i
+      }
+    else
+      ->(pid) { 0 }
+    end
+
+    def memory_usage(pid)
+      MEMORY_GRABBER.call(pid)
+    end
+
+    def to_data
+      @data ||= {
+        "hostname" => hostname,
+        "started_at" => Time.now.to_f,
+        "pid" => ::Process.pid,
+        "tag" => @options[:tag] || "",
+        "concurrency" => @options[:concurrency],
+        "queues" => @options[:queues].uniq,
+        "labels" => @options[:labels],
+        "identity" => identity
+      }
     end
 
     def to_json
-      @json ||= begin
-        # this data changes infrequently so dump it to a string
-        # now so we don't need to dump it every heartbeat.
-        Sidekiq.dump_json(to_data)
-      end
+      # this data changes infrequently so dump it to a string
+      # now so we don't need to dump it every heartbeat.
+      @json ||= Sidekiq.dump_json(to_data)
     end
   end
 end

data/lib/sidekiq/logger.rb

@@ -6,15 +6,20 @@ require "time"
 module Sidekiq
   module Context
     def self.with(hash)
+      orig_context = current.dup
       current.merge!(hash)
       yield
     ensure
-      hash.each_key { |key| current.delete(key) }
+      Thread.current[:sidekiq_context] = orig_context
     end
 
     def self.current
       Thread.current[:sidekiq_context] ||= {}
     end
+
+    def self.add(k, v)
+      Thread.current[:sidekiq_context][k] = v
+    end
   end
 
   module LoggingUtils
@@ -89,7 +94,7 @@ module Sidekiq
       return true if @logdev.nil? || severity < level
 
       if message.nil?
-        if block_given?
+        if block
           message = yield
         else
           message = progname

data/lib/sidekiq/manager.rb

@@ -35,7 +35,7 @@ module Sidekiq
       @done = false
       @workers = Set.new
       @count.times do
-        @workers << Processor.new(self)
+        @workers << Processor.new(self, options)
       end
       @plock = Mutex.new
     end
@@ -55,9 +55,6 @@ module Sidekiq
       fire_event(:quiet, reverse: true)
     end
 
-    # hack for quicker development / testing environment #2774
-    PAUSE_TIME = STDOUT.tty? ? 0.1 : 0.5
-
     def stop(deadline)
       quiet
       fire_event(:shutdown, reverse: true)
@@ -69,12 +66,7 @@ module Sidekiq
       return if @workers.empty?
 
       logger.info { "Pausing to allow workers to finish..." }
-      remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      while remaining > PAUSE_TIME
-        return if @workers.empty?
-        sleep PAUSE_TIME
-        remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      end
+      wait_for(deadline) { @workers.empty? }
       return if @workers.empty?
 
       hard_shutdown
@@ -90,7 +82,7 @@ module Sidekiq
       @plock.synchronize do
         @workers.delete(processor)
         unless @done
-          p = Processor.new(self)
+          p = Processor.new(self, options)
           @workers << p
           p.start
         end
@@ -123,13 +115,19 @@ module Sidekiq
         # contract says that jobs are run AT LEAST once. Process termination
         # is delayed until we're certain the jobs are back in Redis because
         # it is worse to lose a job than to run it twice.
-        strategy = (@options[:fetch] || Sidekiq::BasicFetch)
+        strategy = @options[:fetch]
         strategy.bulk_requeue(jobs, @options)
       end
 
       cleanup.each do |processor|
         processor.kill
       end
+
+      # when this method returns, we immediately call `exit` which may not give
+      # the remaining threads time to run `ensure` blocks, etc. We pause here up
+      # to 3 seconds to give threads a minimal amount of time to run `ensure` blocks.
+      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + 3
+      wait_for(deadline) { @workers.empty? }
     end
   end
 end

data/lib/sidekiq/middleware/chain.rb

@@ -90,12 +90,12 @@ module Sidekiq
       end
 
       def add(klass, *args)
-        remove(klass) if exists?(klass)
+        remove(klass)
         entries << Entry.new(klass, *args)
       end
 
       def prepend(klass, *args)
-        remove(klass) if exists?(klass)
+        remove(klass)
         entries.insert(0, Entry.new(klass, *args))
       end
 
@@ -132,8 +132,8 @@ module Sidekiq
       def invoke(*args)
         return yield if empty?
 
-        chain = retrieve.dup
-        traverse_chain = lambda do
+        chain = retrieve
+        traverse_chain = proc do
           if chain.empty?
             yield
           else
@@ -144,6 +144,8 @@ module Sidekiq
       end
     end
 
+    private
+
     class Entry
       attr_reader :klass
 

data/lib/sidekiq/middleware/current_attributes.rb

@@ -0,0 +1,57 @@
+require "active_support/current_attributes"
+
+module Sidekiq
+  ##
+  # Automatically save and load any current attributes in the execution context
+  # so context attributes "flow" from Rails actions into any associated jobs.
+  # This can be useful for multi-tenancy, i18n locale, timezone, any implicit
+  # per-request attribute. See +ActiveSupport::CurrentAttributes+.
+  #
+  # @example
+  #
+  #   # in your initializer
+  #   require "sidekiq/middleware/current_attributes"
+  #   Sidekiq::CurrentAttributes.persist(Myapp::Current)
+  #
+  module CurrentAttributes
+    class Save
+      def initialize(cattr)
+        @klass = cattr
+      end
+
+      def call(_, job, _, _)
+        attrs = @klass.attributes
+        if job.has_key?("cattr")
+          job["cattr"].merge!(attrs)
+        else
+          job["cattr"] = attrs
+        end
+        yield
+      end
+    end
+
+    class Load
+      def initialize(cattr)
+        @klass = cattr
+      end
+
+      def call(_, job, _, &block)
+        if job.has_key?("cattr")
+          @klass.set(job["cattr"], &block)
+        else
+          yield
+        end
+      end
+    end
+
+    def self.persist(klass)
+      Sidekiq.configure_client do |config|
+        config.client_middleware.add Save, klass
+      end
+      Sidekiq.configure_server do |config|
+        config.client_middleware.add Save, klass
+        config.server_middleware.add Load, klass
+      end
+    end
+  end
+end

data/lib/sidekiq/paginator.rb

@@ -16,22 +16,22 @@ module Sidekiq
 
         case type
         when "zset"
-          total_size, items = conn.multi {
-            conn.zcard(key)
+          total_size, items = conn.multi { |transaction|
+            transaction.zcard(key)
             if rev
-              conn.zrevrange(key, starting, ending, with_scores: true)
+              transaction.zrevrange(key, starting, ending, with_scores: true)
             else
-              conn.zrange(key, starting, ending, with_scores: true)
+              transaction.zrange(key, starting, ending, with_scores: true)
             end
           }
           [current_page, total_size, items]
         when "list"
-          total_size, items = conn.multi {
-            conn.llen(key)
+          total_size, items = conn.multi { |transaction|
+            transaction.llen(key)
             if rev
-              conn.lrange(key, -ending - 1, -starting - 1)
+              transaction.lrange(key, -ending - 1, -starting - 1)
             else
-              conn.lrange(key, starting, ending)
+              transaction.lrange(key, starting, ending)
             end
           }
           items.reverse! if rev

data/lib/sidekiq/processor.rb

@@ -28,15 +28,15 @@ module Sidekiq
     attr_reader :thread
     attr_reader :job
 
-    def initialize(mgr)
+    def initialize(mgr, options)
       @mgr = mgr
       @down = false
       @done = false
       @job = nil
       @thread = nil
-      @strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
-      @reloader = Sidekiq.options[:reloader]
-      @job_logger = (mgr.options[:job_logger] || Sidekiq::JobLogger).new
+      @strategy = options[:fetch]
+      @reloader = options[:reloader] || proc { |&block| block.call }
+      @job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
       @retrier = Sidekiq::JobRetry.new
     end
 

data/lib/sidekiq/rails.rb

@@ -4,6 +4,22 @@ require "sidekiq/worker"
 
 module Sidekiq
   class Rails < ::Rails::Engine
+    class Reloader
+      def initialize(app = ::Rails.application)
+        @app = app
+      end
+
+      def call
+        @app.reloader.wrap do
+          yield
+        end
+      end
+
+      def inspect
+        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
+      end
+    end
+
     # By including the Options module, we allow AJs to directly control sidekiq features
     # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
     # AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
@@ -21,10 +37,19 @@ module Sidekiq
       end
     end
 
+    initializer "sidekiq.rails_logger" do
+      Sidekiq.configure_server do |_|
+        # This is the integration code necessary so that if code uses `Rails.logger.info "Hello"`,
+        # it will appear in the Sidekiq console with all of the job context. See #5021 and
+        # https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
+        unless ::Rails.logger == ::Sidekiq.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
+          ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(::Sidekiq.logger))
+        end
+      end
+    end
+
     # This hook happens after all initializers are run, just before returning
     # from config/environment.rb back to sidekiq/cli.rb.
-    # We have to add the reloader after initialize to see if cache_classes has
-    # been turned on.
     #
     # None of this matters on the client-side, only within the Sidekiq process itself.
     config.after_initialize do
@@ -32,21 +57,5 @@ module Sidekiq
         Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
       end
     end
-
-    class Reloader
-      def initialize(app = ::Rails.application)
-        @app = app
-      end
-
-      def call
-        @app.reloader.wrap do
-          yield
-        end
-      end
-
-      def inspect
-        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
-      end
-    end
   end
 end

data/lib/sidekiq/redis_connection.rb

@@ -8,16 +8,14 @@ module Sidekiq
   class RedisConnection
     class << self
       def create(options = {})
-        options.keys.each do |key|
-          options[key.to_sym] = options.delete(key)
-        end
+        symbolized_options = options.transform_keys(&:to_sym)
 
-        if !options[:url] && (u = determine_redis_provider)
-          options[:url] = u
+        if !symbolized_options[:url] && (u = determine_redis_provider)
+          symbolized_options[:url] = u
         end
 
-        size = if options[:size]
-          options[:size]
+        size = if symbolized_options[:size]
+          symbolized_options[:size]
         elsif Sidekiq.server?
           # Give ourselves plenty of connections.  pool is lazy
           # so we won't create them until we need them.
@@ -30,11 +28,11 @@ module Sidekiq
 
         verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
 
-        pool_timeout = options[:pool_timeout] || 1
-        log_info(options)
+        pool_timeout = symbolized_options[:pool_timeout] || 1
+        log_info(symbolized_options)
 
         ConnectionPool.new(timeout: pool_timeout, size: size) do
-          build_client(options)
+          build_client(symbolized_options)
         end
       end
 
@@ -96,8 +94,11 @@ module Sidekiq
       def log_info(options)
         redacted = "REDACTED"
 
-        # deep clone so we can muck with these options all we want
-        scrubbed_options = Marshal.load(Marshal.dump(options))
+        # Deep clone so we can muck with these options all we want and exclude
+        # params from dump-and-load that may contain objects that Marshal is
+        # unable to safely dump.
+        keys = options.keys - [:logger, :ssl_params]
+        scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
         if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
           uri.password = redacted
           scrubbed_options[:url] = uri.to_s
@@ -124,7 +125,7 @@ module Sidekiq
         # initialization code at all.
         #
         p = ENV["REDIS_PROVIDER"]
-        if p && p =~ /\:/
+        if p && p =~ /:/
           raise <<~EOM
             REDIS_PROVIDER should be set to the name of the variable which contains the Redis URL, not a URL itself.
             Platforms like Heroku will sell addons that publish a *_URL variable.  You need to tell Sidekiq with REDIS_PROVIDER, e.g.:

data/lib/sidekiq/scheduled.rb

@@ -9,29 +9,56 @@ module Sidekiq
     SETS = %w[retry schedule]
 
     class Enq
-      def enqueue_jobs(now = Time.now.to_f.to_s, sorted_sets = SETS)
+      LUA_ZPOPBYSCORE = <<~LUA
+        local key, now = KEYS[1], ARGV[1]
+        local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
+        if jobs[1] then
+          redis.call("zrem", key, jobs[1])
+          return jobs[1]
+        end
+      LUA
+
+      def initialize
+        @done = false
+        @lua_zpopbyscore_sha = nil
+      end
+
+      def enqueue_jobs(sorted_sets = SETS)
         # A job's "score" in Redis is the time at which it should be processed.
         # Just check Redis for the set of jobs with a timestamp before now.
         Sidekiq.redis do |conn|
           sorted_sets.each do |sorted_set|
-            # Get next items in the queue with scores (time to execute) <= now.
-            until (jobs = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 100])).empty?
-              # We need to go through the list one at a time to reduce the risk of something
-              # going wrong between the time jobs are popped from the scheduled queue and when
-              # they are pushed onto a work queue and losing the jobs.
-              jobs.each do |job|
-                # Pop item off the queue and add it to the work queue. If the job can't be popped from
-                # the queue, it's because another process already popped it so we can move on to the
-                # next one.
-                if conn.zrem(sorted_set, job)
-                  Sidekiq::Client.push(Sidekiq.load_json(job))
-                  Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
-                end
-              end
+            # Get next item in the queue with score (time to execute) <= now.
+            # We need to go through the list one at a time to reduce the risk of something
+            # going wrong between the time jobs are popped from the scheduled queue and when
+            # they are pushed onto a work queue and losing the jobs.
+            while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s]))
+              Sidekiq::Client.push(Sidekiq.load_json(job))
+              Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
             end
           end
         end
       end
+
+      def terminate
+        @done = true
+      end
+
+      private
+
+      def zpopbyscore(conn, keys: nil, argv: nil)
+        if @lua_zpopbyscore_sha.nil?
+          raw_conn = conn.respond_to?(:redis) ? conn.redis : conn
+          @lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
+        end
+
+        conn.evalsha(@lua_zpopbyscore_sha, keys: keys, argv: argv)
+      rescue Redis::CommandError => e
+        raise unless e.message.start_with?("NOSCRIPT")
+
+        @lua_zpopbyscore_sha = nil
+        retry
+      end
     end
 
     ##
@@ -49,11 +76,14 @@ module Sidekiq
         @sleeper = ConnectionPool::TimedStack.new
         @done = false
         @thread = nil
+        @count_calls = 0
       end
 
       # Shut down this instance, will pause until the thread is dead.
       def terminate
         @done = true
+        @enq.terminate if @enq.respond_to?(:terminate)
+
         if @thread
           t = @thread
           @thread = nil
@@ -152,8 +182,13 @@ module Sidekiq
       end
 
       def process_count
-        pcount = Sidekiq::ProcessSet.new.size
+        # The work buried within Sidekiq::ProcessSet#cleanup can be
+        # expensive at scale. Cut it down by 90% with this counter.
+        # NB: This method is only called by the scheduler thread so we
+        # don't need to worry about the thread safety of +=.
+        pcount = Sidekiq::ProcessSet.new(@count_calls % 10 == 0).size
         pcount = 1 if pcount == 0
+        @count_calls += 1
         pcount
       end