sidekiq 6.0.0 → 6.3.0

data/lib/sidekiq/launcher.rb

@@ -16,12 +16,13 @@ module Sidekiq
       proc { Sidekiq::VERSION },
       proc { |me, data| data["tag"] },
       proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data["concurrency"]} busy]" },
-      proc { |me, data| "stopping" if me.stopping? },
+      proc { |me, data| "stopping" if me.stopping? }
     ]
 
     attr_accessor :manager, :poller, :fetcher
 
     def initialize(options)
+      options[:fetch] ||= BasicFetch.new(options)
       @manager = Sidekiq::Manager.new(options)
       @poller = Sidekiq::Scheduled::Poller.new
       @done = false
@@ -56,7 +57,7 @@ module Sidekiq
 
       # Requeue everything in case there was a worker who grabbed work while stopped
       # This call is a no-op in Sidekiq but necessary for Sidekiq Pro.
-      strategy = (@options[:fetch] || Sidekiq::BasicFetch)
+      strategy = @options[:fetch]
       strategy.bulk_requeue([], @options)
 
       clear_heartbeat
@@ -68,10 +69,12 @@ module Sidekiq
 
     private unless $TESTING
 
+    BEAT_PAUSE = 5
+
     def start_heartbeat
       loop do
         heartbeat
-        sleep 5
+        sleep BEAT_PAUSE
       end
       Sidekiq.logger.info("Heartbeat stopping...")
     end
@@ -83,7 +86,7 @@ module Sidekiq
       Sidekiq.redis do |conn|
         conn.pipelined do
           conn.srem("processes", identity)
-          conn.del("#{identity}:workers")
+          conn.unlink("#{identity}:workers")
         end
       end
     rescue
@@ -96,6 +99,32 @@ module Sidekiq
       ❤
     end
 
+    def self.flush_stats
+      fails = Processor::FAILURE.reset
+      procd = Processor::PROCESSED.reset
+      return if fails + procd == 0
+
+      nowdate = Time.now.utc.strftime("%Y-%m-%d")
+      begin
+        Sidekiq.redis do |conn|
+          conn.pipelined do
+            conn.incrby("stat:processed", procd)
+            conn.incrby("stat:processed:#{nowdate}", procd)
+            conn.expire("stat:processed:#{nowdate}", STATS_TTL)
+
+            conn.incrby("stat:failed", fails)
+            conn.incrby("stat:failed:#{nowdate}", fails)
+            conn.expire("stat:failed:#{nowdate}", STATS_TTL)
+          end
+        end
+      rescue => ex
+        # we're exiting the process, things might be shut down so don't
+        # try to handle the exception
+        Sidekiq.logger.warn("Unable to flush stats: #{ex}")
+      end
+    end
+    at_exit(&method(:flush_stats))
+
     def ❤
       key = identity
       fails = procd = 0
@@ -118,7 +147,7 @@ module Sidekiq
             conn.incrby("stat:failed:#{nowdate}", fails)
             conn.expire("stat:failed:#{nowdate}", STATS_TTL)
 
-            conn.del(workers_key)
+            conn.unlink(workers_key)
             curstate.each_pair do |tid, hash|
               conn.hset(workers_key, tid, Sidekiq.dump_json(hash))
             end
@@ -126,18 +155,24 @@ module Sidekiq
           end
         end
 
+        rtt = check_rtt
+
         fails = procd = 0
+        kb = memory_usage(::Process.pid)
 
         _, exists, _, _, msg = Sidekiq.redis { |conn|
-          res = conn.multi {
+          conn.multi {
             conn.sadd("processes", key)
-            conn.exists(key)
-            conn.hmset(key, "info", to_json, "busy", curstate.size, "beat", Time.now.to_f, "quiet", @done)
+            conn.exists?(key)
+            conn.hmset(key, "info", to_json,
+              "busy", curstate.size,
+              "beat", Time.now.to_f,
+              "rtt_us", rtt,
+              "quiet", @done,
+              "rss", kb)
             conn.expire(key, 60)
             conn.rpop("#{key}-signals")
           }
-
-          res
         }
 
         # first heartbeat or recovering from an outage and need to reestablish our heartbeat
@@ -148,34 +183,81 @@ module Sidekiq
         ::Process.kill(msg, ::Process.pid)
       rescue => e
         # ignore all redis/network issues
-        logger.error("heartbeat: #{e.message}")
+        logger.error("heartbeat: #{e}")
         # don't lose the counts if there was a network issue
         Processor::PROCESSED.incr(procd)
         Processor::FAILURE.incr(fails)
       end
     end
 
-    def to_data
-      @data ||= begin
-        {
-          "hostname" => hostname,
-          "started_at" => Time.now.to_f,
-          "pid" => ::Process.pid,
-          "tag" => @options[:tag] || "",
-          "concurrency" => @options[:concurrency],
-          "queues" => @options[:queues].uniq,
-          "labels" => @options[:labels],
-          "identity" => identity,
-        }
+    # We run the heartbeat every five seconds.
+    # Capture five samples of RTT, log a warning if each sample
+    # is above our warning threshold.
+    RTT_READINGS = RingBuffer.new(5)
+    RTT_WARNING_LEVEL = 50_000
+
+    def check_rtt
+      a = b = 0
+      Sidekiq.redis do |x|
+        a = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
+        x.ping
+        b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
+      end
+      rtt = b - a
+      RTT_READINGS << rtt
+      # Ideal RTT for Redis is < 1000µs
+      # Workable is < 10,000µs
+      # Log a warning if it's a disaster.
+      if RTT_READINGS.all? { |x| x > RTT_WARNING_LEVEL }
+        Sidekiq.logger.warn <<~EOM
+          Your Redis network connection is performing extremely poorly.
+          Last RTT readings were #{RTT_READINGS.buffer.inspect}, ideally these should be < 1000.
+          Ensure Redis is running in the same AZ or datacenter as Sidekiq.
+          If these values are close to 100,000, that means your Sidekiq process may be
+          CPU overloaded; see https://github.com/mperham/sidekiq/discussions/5039
+        EOM
+        RTT_READINGS.reset
       end
+      rtt
+    end
+
+    MEMORY_GRABBER = case RUBY_PLATFORM
+    when /linux/
+      ->(pid) {
+        IO.readlines("/proc/#{$$}/status").each do |line|
+          next unless line.start_with?("VmRSS:")
+          break line.split[1].to_i
+        end
+      }
+    when /darwin|bsd/
+      ->(pid) {
+        `ps -o pid,rss -p #{pid}`.lines.last.split.last.to_i
+      }
+    else
+      ->(pid) { 0 }
+    end
+
+    def memory_usage(pid)
+      MEMORY_GRABBER.call(pid)
+    end
+
+    def to_data
+      @data ||= {
+        "hostname" => hostname,
+        "started_at" => Time.now.to_f,
+        "pid" => ::Process.pid,
+        "tag" => @options[:tag] || "",
+        "concurrency" => @options[:concurrency],
+        "queues" => @options[:queues].uniq,
+        "labels" => @options[:labels],
+        "identity" => identity
+      }
     end
 
     def to_json
-      @json ||= begin
-        # this data changes infrequently so dump it to a string
-        # now so we don't need to dump it every heartbeat.
-        Sidekiq.dump_json(to_data)
-      end
+      # this data changes infrequently so dump it to a string
+      # now so we don't need to dump it every heartbeat.
+      @json ||= Sidekiq.dump_json(to_data)
     end
   end
 end

data/lib/sidekiq/logger.rb

@@ -4,22 +4,110 @@ require "logger"
 require "time"
 
 module Sidekiq
-  class Logger < ::Logger
-    def initialize(*args)
-      super
+  module Context
+    def self.with(hash)
+      orig_context = current.dup
+      current.merge!(hash)
+      yield
+    ensure
+      Thread.current[:sidekiq_context] = orig_context
+    end
 
-      self.formatter = Sidekiq.log_formatter
+    def self.current
+      Thread.current[:sidekiq_context] ||= {}
+    end
+  end
+
+  module LoggingUtils
+    LEVELS = {
+      "debug" => 0,
+      "info" => 1,
+      "warn" => 2,
+      "error" => 3,
+      "fatal" => 4
+    }
+    LEVELS.default_proc = proc do |_, level|
+      Sidekiq.logger.warn("Invalid log level: #{level.inspect}")
+      nil
+    end
+
+    def debug?
+      level <= 0
+    end
+
+    def info?
+      level <= 1
+    end
+
+    def warn?
+      level <= 2
+    end
+
+    def error?
+      level <= 3
+    end
+
+    def fatal?
+      level <= 4
     end
 
-    def with_context(hash)
-      ctx.merge!(hash)
+    def local_level
+      Thread.current[:sidekiq_log_level]
+    end
+
+    def local_level=(level)
+      case level
+      when Integer
+        Thread.current[:sidekiq_log_level] = level
+      when Symbol, String
+        Thread.current[:sidekiq_log_level] = LEVELS[level.to_s]
+      when nil
+        Thread.current[:sidekiq_log_level] = nil
+      else
+        raise ArgumentError, "Invalid log level: #{level.inspect}"
+      end
+    end
+
+    def level
+      local_level || super
+    end
+
+    # Change the thread-local level for the duration of the given block.
+    def log_at(level)
+      old_local_level = local_level
+      self.local_level = level
       yield
     ensure
-      hash.keys.each { |key| ctx.delete(key) }
+      self.local_level = old_local_level
     end
 
-    def ctx
-      Thread.current[:sidekiq_context] ||= {}
+    # Redefined to check severity against #level, and thus the thread-local level, rather than +@level+.
+    # FIXME: Remove when the minimum Ruby version supports overriding Logger#level.
+    def add(severity, message = nil, progname = nil, &block)
+      severity ||= ::Logger::UNKNOWN
+      progname ||= @progname
+
+      return true if @logdev.nil? || severity < level
+
+      if message.nil?
+        if block
+          message = yield
+        else
+          message = progname
+          progname = @progname
+        end
+      end
+
+      @logdev.write format_message(format_severity(severity), Time.now, progname, message)
+    end
+  end
+
+  class Logger < ::Logger
+    include LoggingUtils
+
+    def initialize(*args, **kwargs)
+      super
+      self.formatter = Sidekiq.log_formatter
     end
 
     module Formatters
@@ -29,11 +117,20 @@ module Sidekiq
         end
 
         def ctx
-          Thread.current[:sidekiq_context] ||= {}
+          Sidekiq::Context.current
         end
 
         def format_context
-          " " + ctx.compact.map { |k, v| "#{k}=#{v}" }.join(" ") if ctx.any?
+          if ctx.any?
+            " " + ctx.compact.map { |k, v|
+              case v
+              when Array
+                "#{k}=#{v.join(",")}"
+              else
+                "#{k}=#{v}"
+              end
+            }.join(" ")
+          end
         end
       end
 
@@ -56,7 +153,7 @@ module Sidekiq
             pid: ::Process.pid,
             tid: tid,
             lvl: severity,
-            msg: message,
+            msg: message
           }
           c = ctx
           hash["ctx"] = c unless c.empty?

data/lib/sidekiq/manager.rb

@@ -35,7 +35,7 @@ module Sidekiq
       @done = false
       @workers = Set.new
       @count.times do
-        @workers << Processor.new(self)
+        @workers << Processor.new(self, options)
       end
       @plock = Mutex.new
     end
@@ -56,7 +56,7 @@ module Sidekiq
     end
 
     # hack for quicker development / testing environment #2774
-    PAUSE_TIME = STDOUT.tty? ? 0.1 : 0.5
+    PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
 
     def stop(deadline)
       quiet
@@ -90,7 +90,7 @@ module Sidekiq
       @plock.synchronize do
         @workers.delete(processor)
         unless @done
-          p = Processor.new(self)
+          p = Processor.new(self, options)
           @workers << p
           p.start
         end
@@ -123,7 +123,7 @@ module Sidekiq
         # contract says that jobs are run AT LEAST once. Process termination
         # is delayed until we're certain the jobs are back in Redis because
         # it is worse to lose a job than to run it twice.
-        strategy = (@options[:fetch] || Sidekiq::BasicFetch)
+        strategy = @options[:fetch]
         strategy.bulk_requeue(jobs, @options)
       end
 

data/lib/sidekiq/middleware/chain.rb

@@ -67,7 +67,6 @@ module Sidekiq
   module Middleware
     class Chain
       include Enumerable
-      attr_reader :entries
 
       def initialize_copy(copy)
         copy.instance_variable_set(:@entries, entries.dup)
@@ -78,21 +77,25 @@ module Sidekiq
       end
 
       def initialize
-        @entries = []
+        @entries = nil
         yield self if block_given?
       end
 
+      def entries
+        @entries ||= []
+      end
+
       def remove(klass)
         entries.delete_if { |entry| entry.klass == klass }
       end
 
       def add(klass, *args)
-        remove(klass) if exists?(klass)
+        remove(klass)
         entries << Entry.new(klass, *args)
       end
 
       def prepend(klass, *args)
-        remove(klass) if exists?(klass)
+        remove(klass)
         entries.insert(0, Entry.new(klass, *args))
       end
 
@@ -114,6 +117,10 @@ module Sidekiq
         any? { |entry| entry.klass == klass }
       end
 
+      def empty?
+        @entries.nil? || @entries.empty?
+      end
+
       def retrieve
         map(&:make_new)
       end
@@ -123,8 +130,10 @@ module Sidekiq
       end
 
       def invoke(*args)
-        chain = retrieve.dup
-        traverse_chain = lambda do
+        return yield if empty?
+
+        chain = retrieve
+        traverse_chain = proc do
           if chain.empty?
             yield
           else
@@ -135,6 +144,8 @@ module Sidekiq
       end
     end
 
+    private
+
     class Entry
       attr_reader :klass
 

data/lib/sidekiq/middleware/current_attributes.rb

@@ -0,0 +1,48 @@
+require "active_support/current_attributes"
+
+module Sidekiq
+  ##
+  # Automatically save and load any current attributes in the execution context
+  # so context attributes "flow" from Rails actions into any associated jobs.
+  # This can be useful for multi-tenancy, i18n locale, timezone, any implicit
+  # per-request attribute. See +ActiveSupport::CurrentAttributes+.
+  #
+  # @example
+  #
+  #   # in your initializer
+  #   require "sidekiq/middleware/current_attributes"
+  #   Sidekiq::CurrentAttributes.persist(Myapp::Current)
+  #
+  module CurrentAttributes
+    class Save
+      def initialize(with:)
+        @klass = with
+      end
+
+      def call(_, job, _, _)
+        job["ctx"] = @klass.attributes
+        yield
+      end
+    end
+
+    class Load
+      def initialize(with:)
+        @klass = with
+      end
+
+      def call(_, job, _, &block)
+        @klass.set(job["ctx"], &block)
+      end
+    end
+
+    def self.persist(klass)
+      Sidekiq.configure_client do |config|
+        config.client_middleware.add Save, with: klass
+      end
+      Sidekiq.configure_server do |config|
+        config.client_middleware.add Save, with: klass
+        config.server_middleware.add Load, with: klass
+      end
+    end
+  end
+end

data/lib/sidekiq/monitor.rb

@@ -4,21 +4,6 @@ require "fileutils"
 require "sidekiq/api"
 
 class Sidekiq::Monitor
-  CMD = File.basename($PROGRAM_NAME)
-
-  attr_reader :stage
-
-  def self.print_usage
-    puts "#{CMD} - monitor Sidekiq from the command line."
-    puts
-    puts "Usage: #{CMD} status <section>"
-    puts
-    puts "       <section> (optional) view a specific section of the status output"
-    puts "       Valid sections are: #{Sidekiq::Monitor::Status::VALID_SECTIONS.join(", ")}"
-    puts
-    puts "Set REDIS_URL to the location of your Redis server if not monitoring localhost."
-  end
-
   class Status
     VALID_SECTIONS = %w[all version overview processes queues]
     COL_PAD = 2
@@ -47,7 +32,7 @@ class Sidekiq::Monitor
 
     def version
       puts "Sidekiq #{Sidekiq::VERSION}"
-      puts Time.now
+      puts Time.now.utc
     end
 
     def overview
@@ -77,7 +62,7 @@ class Sidekiq::Monitor
       columns = {
         name: [:ljust, (["name"] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
         size: [:rjust, (["size"] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
-        latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD],
+        latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
       }
       columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
       puts
@@ -116,7 +101,7 @@ class Sidekiq::Monitor
       tags = [
         process["tag"],
         process["labels"],
-        (process["quiet"] == "true" ? "quiet" : nil),
+        (process["quiet"] == "true" ? "quiet" : nil)
       ].flatten.compact
       tags.any? ? "[#{tags.join("] [")}]" : nil
     end

data/lib/sidekiq/paginator.rb

@@ -12,10 +12,10 @@ module Sidekiq
 
       Sidekiq.redis do |conn|
         type = conn.type(key)
+        rev = opts && opts[:reverse]
 
         case type
         when "zset"
-          rev = opts && opts[:reverse]
           total_size, items = conn.multi {
             conn.zcard(key)
             if rev
@@ -28,8 +28,13 @@ module Sidekiq
         when "list"
           total_size, items = conn.multi {
             conn.llen(key)
-            conn.lrange(key, starting, ending)
+            if rev
+              conn.lrange(key, -ending - 1, -starting - 1)
+            else
+              conn.lrange(key, starting, ending)
+            end
           }
+          items.reverse! if rev
           [current_page, total_size, items]
         when "none"
           [1, 0, []]