sidekiq 6.0.7 → 6.4.2

data/lib/sidekiq/api.rb

@@ -8,7 +8,7 @@ require "base64"
 module Sidekiq
   class Stats
     def initialize
-      fetch_stats!
+      fetch_stats_fast!
     end
 
     def processed
@@ -51,50 +51,33 @@ module Sidekiq
       Sidekiq::Stats::Queues.new.lengths
     end
 
-    def fetch_stats!
+    # O(1) redis calls
+    def fetch_stats_fast!
       pipe1_res = Sidekiq.redis { |conn|
-        conn.pipelined do
-          conn.get("stat:processed")
-          conn.get("stat:failed")
-          conn.zcard("schedule")
-          conn.zcard("retry")
-          conn.zcard("dead")
-          conn.scard("processes")
-          conn.lrange("queue:default", -1, -1)
+        conn.pipelined do |pipeline|
+          pipeline.get("stat:processed")
+          pipeline.get("stat:failed")
+          pipeline.zcard("schedule")
+          pipeline.zcard("retry")
+          pipeline.zcard("dead")
+          pipeline.scard("processes")
+          pipeline.lrange("queue:default", -1, -1)
         end
       }
 
-      processes = Sidekiq.redis { |conn|
-        conn.sscan_each("processes").to_a
-      }
-
-      queues = Sidekiq.redis { |conn|
-        conn.sscan_each("queues").to_a
-      }
-
-      pipe2_res = Sidekiq.redis { |conn|
-        conn.pipelined do
-          processes.each { |key| conn.hget(key, "busy") }
-          queues.each { |queue| conn.llen("queue:#{queue}") }
-        end
-      }
-
-      s = processes.size
-      workers_size = pipe2_res[0...s].sum(&:to_i)
-      enqueued = pipe2_res[s..-1].sum(&:to_i)
-
       default_queue_latency = if (entry = pipe1_res[6].first)
         job = begin
-                Sidekiq.load_json(entry)
-              rescue
-                {}
-              end
+          Sidekiq.load_json(entry)
+        rescue
+          {}
+        end
         now = Time.now.to_f
         thence = job["enqueued_at"] || now
         now - thence
       else
         0
       end
+
       @stats = {
         processed: pipe1_res[0].to_i,
         failed: pipe1_res[1].to_i,
@@ -103,10 +86,39 @@ module Sidekiq
         dead_size: pipe1_res[4],
         processes_size: pipe1_res[5],
 
-        default_queue_latency: default_queue_latency,
-        workers_size: workers_size,
-        enqueued: enqueued
+        default_queue_latency: default_queue_latency
+      }
+    end
+
+    # O(number of processes + number of queues) redis calls
+    def fetch_stats_slow!
+      processes = Sidekiq.redis { |conn|
+        conn.sscan_each("processes").to_a
+      }
+
+      queues = Sidekiq.redis { |conn|
+        conn.sscan_each("queues").to_a
+      }
+
+      pipe2_res = Sidekiq.redis { |conn|
+        conn.pipelined do |pipeline|
+          processes.each { |key| pipeline.hget(key, "busy") }
+          queues.each { |queue| pipeline.llen("queue:#{queue}") }
+        end
       }
+
+      s = processes.size
+      workers_size = pipe2_res[0...s].sum(&:to_i)
+      enqueued = pipe2_res[s..-1].sum(&:to_i)
+
+      @stats[:workers_size] = workers_size
+      @stats[:enqueued] = enqueued
+      @stats
+    end
+
+    def fetch_stats!
+      fetch_stats_fast!
+      fetch_stats_slow!
     end
 
     def reset(*stats)
@@ -126,7 +138,8 @@ module Sidekiq
     private
 
     def stat(s)
-      @stats[s]
+      fetch_stats_slow! if @stats[s].nil?
+      @stats[s] || raise(ArgumentError, "Unknown stat #{s}")
     end
 
     class Queues
@@ -134,20 +147,22 @@ module Sidekiq
         Sidekiq.redis do |conn|
           queues = conn.sscan_each("queues").to_a
 
-          lengths = conn.pipelined {
+          lengths = conn.pipelined { |pipeline|
             queues.each do |queue|
-              conn.llen("queue:#{queue}")
+              pipeline.llen("queue:#{queue}")
             end
           }
 
           array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
-          Hash[array_of_arrays]
+          array_of_arrays.to_h
         end
       end
     end
 
     class History
       def initialize(days_previous, start_date = nil)
+        # we only store five years of data in Redis
+        raise ArgumentError if days_previous < 1 || days_previous > (5 * 365)
         @days_previous = days_previous
         @start_date = start_date || Time.now.utc.to_date
       end
@@ -255,7 +270,7 @@ module Sidekiq
         break if entries.empty?
         page += 1
         entries.each do |entry|
-          yield Job.new(entry, @name)
+          yield JobRecord.new(entry, @name)
         end
         deleted_size = initial_size - size
       end
@@ -265,16 +280,16 @@ module Sidekiq
     # Find the job with the given JID within this queue.
     #
     # This is a slow, inefficient operation.  Do not use under
-    # normal conditions.  Sidekiq Pro contains a faster version.
+    # normal conditions.
     def find_job(jid)
       detect { |j| j.jid == jid }
     end
 
     def clear
       Sidekiq.redis do |conn|
-        conn.multi do
-          conn.unlink(@rname)
-          conn.srem("queues", name)
+        conn.multi do |transaction|
+          transaction.unlink(@rname)
+          transaction.srem("queues", name)
         end
       end
     end
@@ -286,9 +301,9 @@ module Sidekiq
   # sorted set.
   #
   # The job should be considered immutable but may be
-  # removed from the queue via Job#delete.
+  # removed from the queue via JobRecord#delete.
   #
-  class Job
+  class JobRecord
     attr_reader :item
     attr_reader :value
 
@@ -316,48 +331,54 @@ module Sidekiq
 
     def display_class
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
-      @klass ||= case klass
-                 when /\ASidekiq::Extensions::Delayed/
-                   safe_load(args[0], klass) do |target, method, _|
-                     "#{target}.#{method}"
-                   end
-                 when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
-                   job_class = @item["wrapped"] || args[0]
-                   if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
-                     # MailerClass#mailer_method
-                     args[0]["arguments"][0..1].join("#")
-                   else
-                     job_class
-                   end
-                 else
-                   klass
+      @klass ||= self["display_class"] || begin
+        case klass
+        when /\ASidekiq::Extensions::Delayed/
+          safe_load(args[0], klass) do |target, method, _|
+            "#{target}.#{method}"
+          end
+        when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+          job_class = @item["wrapped"] || args[0]
+          if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
+            # MailerClass#mailer_method
+            args[0]["arguments"][0..1].join("#")
+          else
+            job_class
+          end
+        else
+          klass
+        end
       end
     end
 
     def display_args
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
       @display_args ||= case klass
-                when /\ASidekiq::Extensions::Delayed/
-                  safe_load(args[0], args) do |_, _, arg|
-                    arg
-                  end
-                when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
-                  job_args = self["wrapped"] ? args[0]["arguments"] : []
-                  if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
-                    # remove MailerClass, mailer_method and 'deliver_now'
-                    job_args.drop(3)
-                  elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
-                    # remove MailerClass, mailer_method and 'deliver_now'
-                    job_args.drop(3).first["args"]
-                  else
-                    job_args
-                  end
-                else
-                  if self["encrypt"]
-                    # no point in showing 150+ bytes of random garbage
-                    args[-1] = "[encrypted data]"
-                  end
-                  args
+      when /\ASidekiq::Extensions::Delayed/
+        safe_load(args[0], args) do |_, _, arg, kwarg|
+          if !kwarg || kwarg.empty?
+            arg
+          else
+            [arg, kwarg]
+          end
+        end
+      when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+        job_args = self["wrapped"] ? args[0]["arguments"] : []
+        if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
+          # remove MailerClass, mailer_method and 'deliver_now'
+          job_args.drop(3)
+        elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
+          # remove MailerClass, mailer_method and 'deliver_now'
+          job_args.drop(3).first["args"]
+        else
+          job_args
+        end
+      else
+        if self["encrypt"]
+          # no point in showing 150+ bytes of random garbage
+          args[-1] = "[encrypted data]"
+        end
+        args
       end
     end
 
@@ -443,7 +464,7 @@ module Sidekiq
     end
   end
 
-  class SortedEntry < Job
+  class SortedEntry < JobRecord
     attr_reader :score
     attr_reader :parent
 
@@ -502,9 +523,9 @@ module Sidekiq
 
     def remove_job
       Sidekiq.redis do |conn|
-        results = conn.multi {
-          conn.zrangebyscore(parent.name, score, score)
-          conn.zremrangebyscore(parent.name, score, score)
+        results = conn.multi { |transaction|
+          transaction.zrangebyscore(parent.name, score, score)
+          transaction.zremrangebyscore(parent.name, score, score)
         }.first
 
         if results.size == 1
@@ -525,9 +546,9 @@ module Sidekiq
           yield msg if msg
 
           # push the rest back onto the sorted set
-          conn.multi do
+          conn.multi do |transaction|
             nonmatched.each do |message|
-              conn.zadd(parent.name, score.to_f.to_s, message)
+              transaction.zadd(parent.name, score.to_f.to_s, message)
             end
           end
         end
@@ -714,10 +735,10 @@ module Sidekiq
     def kill(message, opts = {})
       now = Time.now.to_f
       Sidekiq.redis do |conn|
-        conn.multi do
-          conn.zadd(name, now.to_s, message)
-          conn.zremrangebyscore(name, "-inf", now - self.class.timeout)
-          conn.zremrangebyrank(name, 0, - self.class.max_jobs)
+        conn.multi do |transaction|
+          transaction.zadd(name, now.to_s, message)
+          transaction.zremrangebyscore(name, "-inf", now - self.class.timeout)
+          transaction.zremrangebyrank(name, 0, - self.class.max_jobs)
         end
       end
 
@@ -765,9 +786,9 @@ module Sidekiq
       count = 0
       Sidekiq.redis do |conn|
         procs = conn.sscan_each("processes").to_a.sort
-        heartbeats = conn.pipelined {
+        heartbeats = conn.pipelined { |pipeline|
           procs.each do |key|
-            conn.hget(key, "info")
+            pipeline.hget(key, "info")
           end
         }
 
@@ -789,21 +810,25 @@ module Sidekiq
         # We're making a tradeoff here between consuming more memory instead of
         # making more roundtrips to Redis, but if you have hundreds or thousands of workers,
         # you'll be happier this way
-        conn.pipelined do
+        conn.pipelined do |pipeline|
           procs.each do |key|
-            conn.hmget(key, "info", "busy", "beat", "quiet")
+            pipeline.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
           end
         end
       }
 
-      result.each do |info, busy, at_s, quiet|
+      result.each do |info, busy, at_s, quiet, rss, rtt|
         # If a process is stopped between when we query Redis for `procs` and
         # when we query for `result`, we will have an item in `result` that is
         # composed of `nil` values.
         next if info.nil?
 
         hash = Sidekiq.load_json(info)
-        yield Process.new(hash.merge("busy" => busy.to_i, "beat" => at_s.to_f, "quiet" => quiet))
+        yield Process.new(hash.merge("busy" => busy.to_i,
+          "beat" => at_s.to_f,
+          "quiet" => quiet,
+          "rss" => rss.to_i,
+          "rtt_us" => rtt.to_i))
       end
     end
 
@@ -815,6 +840,18 @@ module Sidekiq
       Sidekiq.redis { |conn| conn.scard("processes") }
     end
 
+    # Total number of threads available to execute jobs.
+    # For Sidekiq Enterprise customers this number (in production) must be
+    # less than or equal to your licensed concurrency.
+    def total_concurrency
+      sum { |x| x["concurrency"].to_i }
+    end
+
+    def total_rss_in_kb
+      sum { |x| x["rss"].to_i }
+    end
+    alias_method :total_rss, :total_rss_in_kb
+
     # Returns the identity of the current cluster leader or "" if no leader.
     # This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
     # or Sidekiq Pro.
@@ -864,6 +901,10 @@ module Sidekiq
       self["identity"]
     end
 
+    def queues
+      self["queues"]
+    end
+
     def quiet!
       signal("TSTP")
     end
@@ -885,17 +926,17 @@ module Sidekiq
     def signal(sig)
       key = "#{identity}-signals"
       Sidekiq.redis do |c|
-        c.multi do
-          c.lpush(key, sig)
-          c.expire(key, 60)
+        c.multi do |transaction|
+          transaction.lpush(key, sig)
+          transaction.expire(key, 60)
         end
       end
     end
   end
 
   ##
-  # A worker is a thread that is currently processing a job.
-  # Programmatic access to the current active worker set.
+  # The WorkSet stores the work being done by this Sidekiq cluster.
+  # It tracks the process and thread working on each job.
   #
   # WARNING WARNING WARNING
   #
@@ -903,26 +944,27 @@ module Sidekiq
   # If you call #size => 5 and then expect #each to be
   # called 5 times, you're going to have a bad time.
   #
-  #    workers = Sidekiq::Workers.new
-  #    workers.size => 2
-  #    workers.each do |process_id, thread_id, work|
+  #    works = Sidekiq::WorkSet.new
+  #    works.size => 2
+  #    works.each do |process_id, thread_id, work|
   #      # process_id is a unique identifier per Sidekiq process
   #      # thread_id is a unique identifier per thread
   #      # work is a Hash which looks like:
-  #      # { 'queue' => name, 'run_at' => timestamp, 'payload' => msg }
+  #      # { 'queue' => name, 'run_at' => timestamp, 'payload' => job_hash }
   #      # run_at is an epoch Integer.
   #    end
   #
-  class Workers
+  class WorkSet
     include Enumerable
 
-    def each
+    def each(&block)
+      results = []
       Sidekiq.redis do |conn|
         procs = conn.sscan_each("processes").to_a
         procs.sort.each do |key|
-          valid, workers = conn.pipelined {
-            conn.exists(key)
-            conn.hgetall("#{key}:workers")
+          valid, workers = conn.pipelined { |pipeline|
+            pipeline.exists?(key)
+            pipeline.hgetall("#{key}:work")
           }
           next unless valid
           workers.each_pair do |tid, json|
@@ -930,10 +972,12 @@ module Sidekiq
             p = hsh["payload"]
             # avoid breaking API, this is a side effect of the JSON optimization in #4316
             hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
-            yield key, tid, hsh
+            results << [key, tid, hsh]
           end
         end
       end
+
+      results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
     end
 
     # Note that #size is only as accurate as Sidekiq's heartbeat,
@@ -948,13 +992,17 @@ module Sidekiq
         if procs.empty?
           0
         else
-          conn.pipelined {
+          conn.pipelined { |pipeline|
             procs.each do |key|
-              conn.hget(key, "busy")
+              pipeline.hget(key, "busy")
             end
           }.sum(&:to_i)
         end
       end
     end
   end
+  # Since "worker" is a nebulous term, we've deprecated the use of this class name.
+  # Is "worker" a process, a type of job, a thread? Undefined!
+  # WorkSet better describes the data.
+  Workers = WorkSet
 end

data/lib/sidekiq/cli.rb

@@ -20,7 +20,7 @@ module Sidekiq
     attr_accessor :launcher
     attr_accessor :environment
 
-    def parse(args = ARGV)
+    def parse(args = ARGV.dup)
       setup_options(args)
       initialize_logger
       validate!
@@ -33,8 +33,9 @@ module Sidekiq
     # Code within this method is not tested because it alters
     # global process state irreversibly.  PRs which improve the
     # test coverage of Sidekiq::CLI are welcomed.
-    def run
-      boot_system
+    def run(boot_app: true)
+      boot_application if boot_app
+
       if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
         print_banner
       end
@@ -43,9 +44,17 @@ module Sidekiq
       self_read, self_write = IO.pipe
       sigs = %w[INT TERM TTIN TSTP]
       # USR1 and USR2 don't work on the JVM
-      sigs << "USR2" unless jruby?
+      sigs << "USR2" if Sidekiq.pro? && !jruby?
       sigs.each do |sig|
-        trap sig do
+        old_handler = Signal.trap(sig) do
+          if old_handler.respond_to?(:call)
+            begin
+              old_handler.call
+            rescue Exception => exc
+              # signal handlers can't use Logger so puts only
+              puts ["Error in #{sig} handler", exc].inspect
+            end
+          end
           self_write.puts(sig)
         end
       rescue ArgumentError
@@ -58,9 +67,22 @@ module Sidekiq
 
       # touch the connection pool so it is created before we
       # fire startup and start multithreading.
-      ver = Sidekiq.redis_info["redis_version"]
+      info = Sidekiq.redis_info
+      ver = info["redis_version"]
       raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
 
+      maxmemory_policy = info["maxmemory_policy"]
+      if maxmemory_policy != "noeviction"
+        logger.warn <<~EOM
+
+
+          WARNING: Your Redis instance will evict Sidekiq data under heavy load.
+          The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
+          See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
+
+        EOM
+      end
+
       # Since the user can pass us a connection pool explicitly in the initializer, we
       # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
       cursize = Sidekiq.redis_pool.size
@@ -93,8 +115,8 @@ module Sidekiq
       begin
         launcher.run
 
-        while (readable_io = IO.select([self_read]))
-          signal = readable_io.first[0].gets.strip
+        while self_read.wait_readable
+          signal = self_read.gets.strip
           handle_signal(signal)
         end
       rescue Interrupt
@@ -228,8 +250,7 @@ module Sidekiq
       opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
 
       # set defaults
-      opts[:queues] = ["default"] if opts[:queues].nil? || opts[:queues].empty?
-      opts[:strict] = true if opts[:strict].nil?
+      opts[:queues] = ["default"] if opts[:queues].nil?
       opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
 
       # merge with defaults
@@ -240,7 +261,7 @@ module Sidekiq
       Sidekiq.options
     end
 
-    def boot_system
+    def boot_application
       ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
 
       if File.directory?(options[:require])
@@ -274,7 +295,7 @@ module Sidekiq
           (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
         logger.info "=================================================================="
         logger.info "  Please point Sidekiq to a Rails application or a Ruby file  "
-        logger.info "  to load your worker classes with -r [DIR|FILE]."
+        logger.info "  to load your job classes with -r [DIR|FILE]."
         logger.info "=================================================================="
         logger.info @parser
         die(1)
@@ -315,7 +336,7 @@ module Sidekiq
           parse_queue opts, queue, weight
         end
 
-        o.on "-r", "--require [PATH|DIR]", "Location of Rails application with workers or file to require" do |arg|
+        o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
           opts[:require] = arg
         end
 
@@ -359,7 +380,9 @@ module Sidekiq
     end
 
     def parse_config(path)
-      opts = YAML.load(ERB.new(File.read(path)).result) || {}
+      erb = ERB.new(File.read(path))
+      erb.filename = File.expand_path(path)
+      opts = load_yaml(erb.result) || {}
 
       if opts.respond_to? :deep_symbolize_keys!
         opts.deep_symbolize_keys!
@@ -368,19 +391,30 @@ module Sidekiq
       end
 
       opts = opts.merge(opts.delete(environment.to_sym) || {})
+      opts.delete(:strict)
+
       parse_queues(opts, opts.delete(:queues) || [])
 
       opts
     end
 
+    def load_yaml(src)
+      if Psych::VERSION > "4.0"
+        YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
+      else
+        YAML.load(src)
+      end
+    end
+
     def parse_queues(opts, queues_and_weights)
       queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
     end
 
     def parse_queue(opts, queue, weight = nil)
       opts[:queues] ||= []
+      opts[:strict] = true if opts[:strict].nil?
       raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
-      [weight.to_i, 1].max.times { opts[:queues] << queue }
+      [weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
       opts[:strict] = false if weight.to_i > 0
     end