sidekiq 5.1.3 → 5.2.10

data/lib/sidekiq/api.rb

@@ -2,7 +2,23 @@
 require 'sidekiq'
 
 module Sidekiq
+
+  module RedisScanner
+    def sscan(conn, key)
+      cursor = '0'
+      result = []
+      loop do
+        cursor, values = conn.sscan(key, cursor)
+        result.push(*values)
+        break if cursor == '0'
+      end
+      result
+    end
+  end
+
   class Stats
+    include RedisScanner
+
     def initialize
       fetch_stats!
     end
@@ -57,19 +73,25 @@ module Sidekiq
           conn.zcard('dead')
           conn.scard('processes')
           conn.lrange('queue:default', -1, -1)
-          conn.smembers('processes')
-          conn.smembers('queues')
         end
       end
 
+      processes = Sidekiq.redis do |conn|
+        sscan(conn, 'processes')
+      end
+
+      queues = Sidekiq.redis do |conn|
+        sscan(conn, 'queues')
+      end
+
       pipe2_res = Sidekiq.redis do |conn|
         conn.pipelined do
-          pipe1_res[7].each {|key| conn.hget(key, 'busy') }
-          pipe1_res[8].each {|queue| conn.llen("queue:#{queue}") }
+          processes.each {|key| conn.hget(key, 'busy') }
+          queues.each {|queue| conn.llen("queue:#{queue}") }
         end
       end
 
-      s = pipe1_res[7].size
+      s = processes.size
       workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
       enqueued     = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
 
@@ -116,9 +138,11 @@ module Sidekiq
     end
 
     class Queues
+      include RedisScanner
+
       def lengths
         Sidekiq.redis do |conn|
-          queues = conn.smembers('queues')
+          queues = sscan(conn, 'queues')
 
           lengths = conn.pipelined do
             queues.each do |queue|
@@ -140,6 +164,8 @@ module Sidekiq
 
     class History
       def initialize(days_previous, start_date = nil)
+        #we only store five years of data in Redis
+        raise ArgumentError if days_previous < 1 || days_previous > (5 * 365)
         @days_previous = days_previous
         @start_date = start_date || Time.now.utc.to_date
       end
@@ -198,18 +224,19 @@ module Sidekiq
   #
   class Queue
     include Enumerable
+    extend RedisScanner
 
     ##
     # Return all known queues within Redis.
     #
     def self.all
-      Sidekiq.redis { |c| c.smembers('queues') }.sort.map { |q| Sidekiq::Queue.new(q) }
+      Sidekiq.redis { |c| sscan(c, 'queues') }.sort.map { |q| Sidekiq::Queue.new(q) }
     end
 
     attr_reader :name
 
     def initialize(name="default")
-      @name = name
+      @name = name.to_s
       @rname = "queue:#{name}"
     end
 
@@ -645,6 +672,12 @@ module Sidekiq
         each(&:retry)
       end
     end
+
+    def kill_all
+      while size > 0
+        each(&:kill)
+      end
+    end
   end
 
   ##
@@ -700,17 +733,18 @@ module Sidekiq
   #
   class ProcessSet
     include Enumerable
+    include RedisScanner
 
     def initialize(clean_plz=true)
-      self.class.cleanup if clean_plz
+      cleanup if clean_plz
     end
 
     # Cleans up dead processes recorded in Redis.
     # Returns the number of processes cleaned.
-    def self.cleanup
+    def cleanup
       count = 0
       Sidekiq.redis do |conn|
-        procs = conn.smembers('processes').sort
+        procs = sscan(conn, 'processes').sort
         heartbeats = conn.pipelined do
           procs.each do |key|
             conn.hget(key, 'info')
@@ -730,7 +764,7 @@ module Sidekiq
     end
 
     def each
-      procs = Sidekiq.redis { |conn| conn.smembers('processes') }.sort
+      procs = Sidekiq.redis { |conn| sscan(conn, 'processes') }.sort
 
       Sidekiq.redis do |conn|
         # We're making a tradeoff here between consuming more memory instead of
@@ -865,13 +899,14 @@ module Sidekiq
   #
   class Workers
     include Enumerable
+    include RedisScanner
 
     def each
       Sidekiq.redis do |conn|
-        procs = conn.smembers('processes')
+        procs = sscan(conn, 'processes')
         procs.sort.each do |key|
           valid, workers = conn.pipelined do
-            conn.exists(key)
+            conn.exists?(key)
             conn.hgetall("#{key}:workers")
           end
           next unless valid
@@ -890,7 +925,7 @@ module Sidekiq
     # which can easily get out of sync with crashy processes.
     def size
       Sidekiq.redis do |conn|
-        procs = conn.smembers('processes')
+        procs = sscan(conn, 'processes')
         if procs.empty?
           0
         else

data/lib/sidekiq/cli.rb

@@ -9,6 +9,7 @@ require 'fileutils'
 
 require 'sidekiq'
 require 'sidekiq/util'
+require 'sidekiq/launcher'
 
 module Sidekiq
   class CLI
@@ -23,23 +24,13 @@ module Sidekiq
       proc { |me, data| "stopping" if me.stopping? },
     ]
 
-    # Used for CLI testing
-    attr_accessor :code
     attr_accessor :launcher
     attr_accessor :environment
 
-    def initialize
-      @code = nil
-    end
-
-    def parse(args=ARGV)
-      @code = nil
-
+    def parse(args = ARGV)
       setup_options(args)
       initialize_logger
       validate!
-      daemonize
-      write_pid
     end
 
     def jruby?
@@ -50,8 +41,10 @@ module Sidekiq
     # global process state irreversibly.  PRs which improve the
     # test coverage of Sidekiq::CLI are welcomed.
     def run
+      daemonize if options[:daemon]
+      write_pid
       boot_system
-      print_banner
+      print_banner if environment == 'development' && $stdout.tty?
 
       self_read, self_write = IO.pipe
       sigs = %w(INT TERM TTIN TSTP)
@@ -79,6 +72,13 @@ module Sidekiq
       # fire startup and start multithreading.
       ver = Sidekiq.redis_info['redis_version']
       raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
+      logger.warn "Sidekiq 6.0 will require Redis 4.0+, you are using Redis v#{ver}" if ver < '4'
+
+      # Since the user can pass us a connection pool explicitly in the initializer, we
+      # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
+      cursize = Sidekiq.redis_pool.size
+      needed = Sidekiq.options[:concurrency] + 2
+      raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
 
       # cache process identity
       Sidekiq.options[:identity] = identity
@@ -93,11 +93,14 @@ module Sidekiq
       logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(', ')}" }
       logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}" }
 
+      launch(self_read)
+    end
+
+    def launch(self_read)
       if !options[:daemon]
         logger.info 'Starting processing, hit Ctrl-C to stop'
       end
 
-      require 'sidekiq/launcher'
       @launcher = Sidekiq::Launcher.new(options)
 
       begin
@@ -179,23 +182,15 @@ module Sidekiq
     private
 
     def print_banner
-      # Print logo and banner for development
-      if environment == 'development' && $stdout.tty?
-        puts "\e[#{31}m"
-        puts Sidekiq::CLI.banner
-        puts "\e[0m"
-      end
+      puts "\e[#{31}m"
+      puts Sidekiq::CLI.banner
+      puts "\e[0m"
     end
 
     def daemonize
-      return unless options[:daemon]
-
       raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
-      files_to_reopen = []
-      ObjectSpace.each_object(File) do |file|
-        files_to_reopen << file unless file.closed?
-      end
 
+      files_to_reopen = ObjectSpace.each_object(File).reject { |f| f.closed? }
       ::Process.daemon(true, true)
 
       files_to_reopen.each do |file|
@@ -233,15 +228,38 @@ module Sidekiq
     alias_method :☠, :exit
 
     def setup_options(args)
+      # parse CLI options
       opts = parse_options(args)
+
       set_environment opts[:environment]
 
-      cfile = opts[:config_file]
-      opts = parse_config(cfile).merge(opts) if cfile
+      # check config file presence
+      if opts[:config_file]
+        if opts[:config_file] && !File.exist?(opts[:config_file])
+          raise ArgumentError, "No such file #{opts[:config_file]}"
+        end
+      else
+        config_dir = if File.directory?(opts[:require].to_s)
+          File.join(opts[:require], 'config')
+        else
+          File.join(options[:require], 'config')
+        end
 
+        %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
+          path = File.join(config_dir, config_file)
+          opts[:config_file] ||= path if File.exist?(path)
+        end
+      end
+
+      # parse config file options
+      opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
+
+      # set defaults
+      opts[:queues] = Array(opts[:queues]) << 'default' if opts[:queues].nil? || opts[:queues].empty?
       opts[:strict] = true if opts[:strict].nil?
-      opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"]
+      opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
 
+      # merge with defaults
       options.merge!(opts)
     end
 
@@ -252,8 +270,6 @@ module Sidekiq
     def boot_system
       ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
 
-      raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
-
       if File.directory?(options[:require])
         require 'rails'
         if ::Rails::VERSION::MAJOR < 4
@@ -273,10 +289,7 @@ module Sidekiq
         end
         options[:tag] ||= default_tag
       else
-        not_required_message = "#{options[:require]} was not required, you should use an explicit path: " +
-            "./#{options[:require]} or /path/to/#{options[:require]}"
-
-        require(options[:require]) || raise(ArgumentError, not_required_message)
+        require options[:require]
       end
     end
 
@@ -292,8 +305,6 @@ module Sidekiq
     end
 
     def validate!
-      options[:queues] << 'default' if options[:queues].empty?
-
       if !File.exist?(options[:require]) ||
          (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
         logger.info "=================================================================="
@@ -319,6 +330,7 @@ module Sidekiq
 
         o.on '-d', '--daemon', "Daemonize process" do |arg|
           opts[:daemon] = arg
+          puts "WARNING: Daemonization mode will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services"
         end
 
         o.on '-e', '--environment ENV', "Application environment" do |arg|
@@ -358,10 +370,12 @@ module Sidekiq
 
         o.on '-L', '--logfile PATH', "path to writable logfile" do |arg|
           opts[:logfile] = arg
+          puts "WARNING: Logfile redirection will be removed in Sidekiq 6.0, see #4045. Sidekiq will only log to STDOUT"
         end
 
         o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
           opts[:pidfile] = arg
+          puts "WARNING: PID file creation will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services"
         end
 
         o.on '-V', '--version', "Print version and exit" do |arg|
@@ -375,11 +389,8 @@ module Sidekiq
         logger.info @parser
         die 1
       end
-      @parser.parse!(argv)
 
-      %w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename|
-        opts[:config_file] ||= filename if File.exist?(filename)
-      end
+      @parser.parse!(argv)
 
       opts
     end
@@ -399,23 +410,18 @@ module Sidekiq
       end
     end
 
-    def parse_config(cfile)
-      opts = {}
-      if File.exist?(cfile)
-        opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
-
-        if opts.respond_to? :deep_symbolize_keys!
-          opts.deep_symbolize_keys!
-        else
-          symbolize_keys_deep!(opts)
-        end
+    def parse_config(path)
+      opts = YAML.load(ERB.new(File.read(path)).result) || {}
 
-        opts = opts.merge(opts.delete(environment.to_sym) || {})
-        parse_queues(opts, opts.delete(:queues) || [])
+      if opts.respond_to? :deep_symbolize_keys!
+        opts.deep_symbolize_keys!
       else
-        # allow a non-existent config file so Sidekiq
-        # can be deployed by cap with just the defaults.
+        symbolize_keys_deep!(opts)
       end
+
+      opts = opts.merge(opts.delete(environment.to_sym) || {})
+      parse_queues(opts, opts.delete(:queues) || [])
+
       ns = opts.delete(:namespace)
       if ns
         # logger hasn't been initialized yet, puts is all we have.
@@ -429,10 +435,10 @@ module Sidekiq
       queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
     end
 
-    def parse_queue(opts, q, weight=nil)
-      [weight.to_i, 1].max.times do
-       (opts[:queues] ||= []) << q
-      end
+    def parse_queue(opts, queue, weight = nil)
+      opts[:queues] ||= []
+      raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
+      [weight.to_i, 1].max.times { opts[:queues] << queue }
       opts[:strict] = false if weight.to_i > 0
     end
   end

data/lib/sidekiq/client.rb

@@ -77,9 +77,10 @@ module Sidekiq
     end
 
     ##
-    # Push a large number of jobs to Redis.  In practice this method is only
-    # useful if you are pushing thousands of jobs or more.  This method
-    # cuts out the redis network round trip latency.
+    # Push a large number of jobs to Redis. This method cuts out the redis
+    # network round trip latency.  I wouldn't recommend pushing more than
+    # 1000 per call but YMMV based on network quality, size of job args, etc.
+    # A large number of jobs can cause a bit of Redis command processing latency.
     #
     # Takes the same arguments as #push except that args is expected to be
     # an Array of Arrays.  All other keys are duplicated for each job.  Each job

data/lib/sidekiq/ctl.rb

@@ -0,0 +1,221 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+require 'sidekiq/api'
+
+class Sidekiq::Ctl
+  DEFAULT_KILL_TIMEOUT = 10
+  CMD = File.basename($0)
+
+  attr_reader :stage, :pidfile, :kill_timeout
+
+  def self.print_usage
+    puts "#{CMD} - control Sidekiq from the command line."
+    puts
+    puts "Usage: #{CMD} quiet <pidfile> <kill_timeout>"
+    puts "       #{CMD} stop <pidfile> <kill_timeout>"
+    puts "       #{CMD} status <section>"
+    puts
+    puts "       <pidfile> is path to a pidfile"
+    puts "       <kill_timeout> is number of seconds to wait until Sidekiq exits"
+    puts "       (default: #{Sidekiq::Ctl::DEFAULT_KILL_TIMEOUT}), after which Sidekiq will be KILL'd"
+    puts
+    puts "       <section> (optional) view a specific section of the status output"
+    puts "       Valid sections are: #{Sidekiq::Ctl::Status::VALID_SECTIONS.join(', ')}"
+    puts
+    puts "Be sure to set the kill_timeout LONGER than Sidekiq's -t timeout.  If you want"
+    puts "to wait 60 seconds for jobs to finish, use `sidekiq -t 60` and `sidekiqctl stop"
+    puts " path_to_pidfile 61`"
+    puts
+  end
+
+  def initialize(stage, pidfile, timeout)
+    @stage = stage
+    @pidfile = pidfile
+    @kill_timeout = timeout
+
+    done('No pidfile given', :error) if !pidfile
+    done("Pidfile #{pidfile} does not exist", :warn) if !File.exist?(pidfile)
+    done('Invalid pidfile content', :error) if pid == 0
+
+    fetch_process
+
+    begin
+      send(stage)
+    rescue NoMethodError
+      done "Invalid command: #{stage}", :error
+    end
+  end
+
+  def fetch_process
+    Process.kill(0, pid)
+  rescue Errno::ESRCH
+    done "Process doesn't exist", :error
+  # We were not allowed to send a signal, but the process must have existed
+  # when Process.kill() was called.
+  rescue Errno::EPERM
+    return pid
+  end
+
+  def done(msg, error = nil)
+    puts msg
+    exit(exit_signal(error))
+  end
+
+  def exit_signal(error)
+    (error == :error) ? 1 : 0
+  end
+
+  def pid
+    @pid ||= File.read(pidfile).to_i
+  end
+
+  def quiet
+    `kill -TSTP #{pid}`
+  end
+
+  def stop
+    `kill -TERM #{pid}`
+    kill_timeout.times do
+      begin
+        Process.kill(0, pid)
+      rescue Errno::ESRCH
+        FileUtils.rm_f pidfile
+        done 'Sidekiq shut down gracefully.'
+      rescue Errno::EPERM
+        done 'Not permitted to shut down Sidekiq.'
+      end
+      sleep 1
+    end
+    `kill -9 #{pid}`
+    FileUtils.rm_f pidfile
+    done 'Sidekiq shut down forcefully.'
+  end
+  alias_method :shutdown, :stop
+
+  class Status
+    VALID_SECTIONS = %w[all version overview processes queues]
+    def display(section = nil)
+      section ||= 'all'
+      unless VALID_SECTIONS.include? section
+        puts "I don't know how to check the status of '#{section}'!"
+        puts "Try one of these: #{VALID_SECTIONS.join(', ')}"
+        return
+      end
+      send(section)
+    rescue StandardError => e
+      puts "Couldn't get status: #{e}"
+    end
+
+    def all
+      version
+      puts
+      overview
+      puts
+      processes
+      puts
+      queues
+    end
+
+    def version
+      puts "Sidekiq #{Sidekiq::VERSION}"
+      puts Time.now
+    end
+
+    def overview
+      puts '---- Overview ----'
+      puts "  Processed: #{delimit stats.processed}"
+      puts "     Failed: #{delimit stats.failed}"
+      puts "       Busy: #{delimit stats.workers_size}"
+      puts "   Enqueued: #{delimit stats.enqueued}"
+      puts "    Retries: #{delimit stats.retry_size}"
+      puts "  Scheduled: #{delimit stats.scheduled_size}"
+      puts "       Dead: #{delimit stats.dead_size}"
+    end
+
+    def processes
+      puts "---- Processes (#{process_set.size}) ----"
+      process_set.each_with_index do |process, index|
+        puts "#{process['identity']} #{tags_for(process)}"
+        puts "  Started: #{Time.at(process['started_at'])} (#{time_ago(process['started_at'])})"
+        puts "  Threads: #{process['concurrency']} (#{process['busy']} busy)"
+        puts "   Queues: #{split_multiline(process['queues'].sort, pad: 11)}"
+        puts '' unless (index+1) == process_set.size
+      end
+    end
+
+    COL_PAD = 2
+    def queues
+      puts "---- Queues (#{queue_data.size}) ----"
+      columns = {
+        name: [:ljust, (['name'] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
+        size: [:rjust, (['size'] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
+        latency: [:rjust, (['latency'] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
+      }
+      columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
+      puts
+      queue_data.each do |q|
+        columns.each do |col, (dir, width)|
+          print q.send(col).public_send(dir, width)
+        end
+        puts
+      end
+    end
+
+    private
+
+    def delimit(number)
+      number.to_s.reverse.scan(/.{1,3}/).join(',').reverse
+    end
+
+    def split_multiline(values, opts = {})
+      return 'none' unless values
+      pad = opts[:pad] || 0
+      max_length = opts[:max_length] || (80 - pad)
+      out = []
+      line = ''
+      values.each do |value|
+        if (line.length + value.length) > max_length
+          out << line
+          line = ' ' * pad
+        end
+        line << value + ', '
+      end
+      out << line[0..-3]
+      out.join("\n")
+    end
+
+    def tags_for(process)
+      tags = [
+        process['tag'],
+        process['labels'],
+        (process['quiet'] == 'true' ? 'quiet' : nil)
+      ].flatten.compact
+      tags.any? ? "[#{tags.join('] [')}]" : nil
+    end
+
+    def time_ago(timestamp)
+      seconds = Time.now - Time.at(timestamp)
+      return 'just now' if seconds < 60
+      return 'a minute ago' if seconds < 120
+      return "#{seconds.floor / 60} minutes ago" if seconds < 3600
+      return 'an hour ago' if seconds < 7200
+      "#{seconds.floor / 60 / 60} hours ago"
+    end
+
+    QUEUE_STRUCT = Struct.new(:name, :size, :latency)
+    def queue_data
+      @queue_data ||= Sidekiq::Queue.all.map do |q|
+        QUEUE_STRUCT.new(q.name, q.size.to_s, sprintf('%#.2f', q.latency))
+      end
+    end
+
+    def process_set
+      @process_set ||= Sidekiq::ProcessSet.new
+    end
+
+    def stats
+      @stats ||= Sidekiq::Stats.new
+    end
+  end
+end

data/lib/sidekiq/job_logger.rb

@@ -3,7 +3,7 @@ module Sidekiq
   class JobLogger
 
     def call(item, queue)
-      start = Time.now
+      start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       logger.info("start")
       yield
       logger.info("done: #{elapsed(start)} sec")
@@ -15,7 +15,7 @@ module Sidekiq
     private
 
     def elapsed(start)
-      (Time.now - start).round(3)
+      (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start).round(3)
     end
 
     def logger