sidekiq 5.1.3 → 5.2.9

data/bin/sidekiqload

@@ -65,7 +65,7 @@ def handle_signal(launcher, sig)
     # http://jira.codehaus.org/browse/JRUBY-4637
     raise Interrupt
   when 'TERM'
-    # Heroku sends TERM and then waits 10 seconds for process to exit.
+    # Heroku sends TERM and then waits 30 seconds for process to exit.
     raise Interrupt
   when 'TSTP'
     Sidekiq.logger.info "Received TSTP, no longer accepting new work"

data/lib/sidekiq.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'sidekiq/version'
 fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.2.2." if RUBY_PLATFORM != 'java' && Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.2')
 
@@ -17,7 +18,7 @@ module Sidekiq
   DEFAULTS = {
     queues: [],
     labels: [],
-    concurrency: 25,
+    concurrency: 10,
     require: '.',
     environment: nil,
     timeout: 8,
@@ -56,6 +57,7 @@ module Sidekiq
   def self.options
     @options ||= DEFAULTS.dup
   end
+
   def self.options=(opts)
     @options = opts
   end
@@ -94,8 +96,8 @@ module Sidekiq
       begin
         yield conn
       rescue Redis::CommandError => ex
-        #2550 Failover can cause the server to become a slave, need
-        # to disconnect and reopen the socket to get back to the master.
+        #2550 Failover can cause the server to become a replica, need
+        # to disconnect and reopen the socket to get back to the primary.
         (conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/
         raise
       end

data/lib/sidekiq/api.rb

@@ -2,7 +2,23 @@
 require 'sidekiq'
 
 module Sidekiq
+
+  module RedisScanner
+    def sscan(conn, key)
+      cursor = '0'
+      result = []
+      loop do
+        cursor, values = conn.sscan(key, cursor)
+        result.push(*values)
+        break if cursor == '0'
+      end
+      result
+    end
+  end
+
   class Stats
+    include RedisScanner
+
     def initialize
       fetch_stats!
     end
@@ -57,19 +73,25 @@ module Sidekiq
           conn.zcard('dead')
           conn.scard('processes')
           conn.lrange('queue:default', -1, -1)
-          conn.smembers('processes')
-          conn.smembers('queues')
         end
       end
 
+      processes = Sidekiq.redis do |conn|
+        sscan(conn, 'processes')
+      end
+
+      queues = Sidekiq.redis do |conn|
+        sscan(conn, 'queues')
+      end
+
       pipe2_res = Sidekiq.redis do |conn|
         conn.pipelined do
-          pipe1_res[7].each {|key| conn.hget(key, 'busy') }
-          pipe1_res[8].each {|queue| conn.llen("queue:#{queue}") }
+          processes.each {|key| conn.hget(key, 'busy') }
+          queues.each {|queue| conn.llen("queue:#{queue}") }
         end
       end
 
-      s = pipe1_res[7].size
+      s = processes.size
       workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
       enqueued     = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
 
@@ -116,9 +138,11 @@ module Sidekiq
     end
 
     class Queues
+      include RedisScanner
+
       def lengths
         Sidekiq.redis do |conn|
-          queues = conn.smembers('queues')
+          queues = sscan(conn, 'queues')
 
           lengths = conn.pipelined do
             queues.each do |queue|
@@ -198,18 +222,19 @@ module Sidekiq
   #
   class Queue
     include Enumerable
+    extend RedisScanner
 
     ##
     # Return all known queues within Redis.
     #
     def self.all
-      Sidekiq.redis { |c| c.smembers('queues') }.sort.map { |q| Sidekiq::Queue.new(q) }
+      Sidekiq.redis { |c| sscan(c, 'queues') }.sort.map { |q| Sidekiq::Queue.new(q) }
     end
 
     attr_reader :name
 
     def initialize(name="default")
-      @name = name
+      @name = name.to_s
       @rname = "queue:#{name}"
     end
 
@@ -645,6 +670,12 @@ module Sidekiq
         each(&:retry)
       end
     end
+
+    def kill_all
+      while size > 0
+        each(&:kill)
+      end
+    end
   end
 
   ##
@@ -700,17 +731,18 @@ module Sidekiq
   #
   class ProcessSet
     include Enumerable
+    include RedisScanner
 
     def initialize(clean_plz=true)
-      self.class.cleanup if clean_plz
+      cleanup if clean_plz
     end
 
     # Cleans up dead processes recorded in Redis.
     # Returns the number of processes cleaned.
-    def self.cleanup
+    def cleanup
       count = 0
       Sidekiq.redis do |conn|
-        procs = conn.smembers('processes').sort
+        procs = sscan(conn, 'processes').sort
         heartbeats = conn.pipelined do
           procs.each do |key|
             conn.hget(key, 'info')
@@ -730,7 +762,7 @@ module Sidekiq
     end
 
     def each
-      procs = Sidekiq.redis { |conn| conn.smembers('processes') }.sort
+      procs = Sidekiq.redis { |conn| sscan(conn, 'processes') }.sort
 
       Sidekiq.redis do |conn|
         # We're making a tradeoff here between consuming more memory instead of
@@ -865,10 +897,11 @@ module Sidekiq
   #
   class Workers
     include Enumerable
+    include RedisScanner
 
     def each
       Sidekiq.redis do |conn|
-        procs = conn.smembers('processes')
+        procs = sscan(conn, 'processes')
         procs.sort.each do |key|
           valid, workers = conn.pipelined do
             conn.exists(key)
@@ -890,7 +923,7 @@ module Sidekiq
     # which can easily get out of sync with crashy processes.
     def size
       Sidekiq.redis do |conn|
-        procs = conn.smembers('processes')
+        procs = sscan(conn, 'processes')
         if procs.empty?
           0
         else

data/lib/sidekiq/cli.rb

@@ -9,6 +9,7 @@ require 'fileutils'
 
 require 'sidekiq'
 require 'sidekiq/util'
+require 'sidekiq/launcher'
 
 module Sidekiq
   class CLI
@@ -23,23 +24,13 @@ module Sidekiq
       proc { |me, data| "stopping" if me.stopping? },
     ]
 
-    # Used for CLI testing
-    attr_accessor :code
     attr_accessor :launcher
     attr_accessor :environment
 
-    def initialize
-      @code = nil
-    end
-
-    def parse(args=ARGV)
-      @code = nil
-
+    def parse(args = ARGV)
       setup_options(args)
       initialize_logger
       validate!
-      daemonize
-      write_pid
     end
 
     def jruby?
@@ -50,8 +41,10 @@ module Sidekiq
     # global process state irreversibly.  PRs which improve the
     # test coverage of Sidekiq::CLI are welcomed.
     def run
+      daemonize if options[:daemon]
+      write_pid
       boot_system
-      print_banner
+      print_banner if environment == 'development' && $stdout.tty?
 
       self_read, self_write = IO.pipe
       sigs = %w(INT TERM TTIN TSTP)
@@ -79,6 +72,13 @@ module Sidekiq
       # fire startup and start multithreading.
       ver = Sidekiq.redis_info['redis_version']
       raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
+      logger.warn "Sidekiq 6.0 will require Redis 4.0+, you are using Redis v#{ver}" if ver < '4'
+
+      # Since the user can pass us a connection pool explicitly in the initializer, we
+      # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
+      cursize = Sidekiq.redis_pool.size
+      needed = Sidekiq.options[:concurrency] + 2
+      raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
 
       # cache process identity
       Sidekiq.options[:identity] = identity
@@ -93,11 +93,14 @@ module Sidekiq
       logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(', ')}" }
       logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}" }
 
+      launch(self_read)
+    end
+
+    def launch(self_read)
       if !options[:daemon]
         logger.info 'Starting processing, hit Ctrl-C to stop'
       end
 
-      require 'sidekiq/launcher'
       @launcher = Sidekiq::Launcher.new(options)
 
       begin
@@ -179,23 +182,15 @@ module Sidekiq
     private
 
     def print_banner
-      # Print logo and banner for development
-      if environment == 'development' && $stdout.tty?
-        puts "\e[#{31}m"
-        puts Sidekiq::CLI.banner
-        puts "\e[0m"
-      end
+      puts "\e[#{31}m"
+      puts Sidekiq::CLI.banner
+      puts "\e[0m"
     end
 
     def daemonize
-      return unless options[:daemon]
-
       raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
-      files_to_reopen = []
-      ObjectSpace.each_object(File) do |file|
-        files_to_reopen << file unless file.closed?
-      end
 
+      files_to_reopen = ObjectSpace.each_object(File).reject { |f| f.closed? }
       ::Process.daemon(true, true)
 
       files_to_reopen.each do |file|
@@ -233,15 +228,38 @@ module Sidekiq
     alias_method :☠, :exit
 
     def setup_options(args)
+      # parse CLI options
       opts = parse_options(args)
+
       set_environment opts[:environment]
 
-      cfile = opts[:config_file]
-      opts = parse_config(cfile).merge(opts) if cfile
+      # check config file presence
+      if opts[:config_file]
+        if opts[:config_file] && !File.exist?(opts[:config_file])
+          raise ArgumentError, "No such file #{opts[:config_file]}"
+        end
+      else
+        config_dir = if File.directory?(opts[:require].to_s)
+          File.join(opts[:require], 'config')
+        else
+          File.join(options[:require], 'config')
+        end
 
+        %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
+          path = File.join(config_dir, config_file)
+          opts[:config_file] ||= path if File.exist?(path)
+        end
+      end
+
+      # parse config file options
+      opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
+
+      # set defaults
+      opts[:queues] = Array(opts[:queues]) << 'default' if opts[:queues].nil? || opts[:queues].empty?
       opts[:strict] = true if opts[:strict].nil?
-      opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"]
+      opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
 
+      # merge with defaults
       options.merge!(opts)
     end
 
@@ -252,8 +270,6 @@ module Sidekiq
     def boot_system
       ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
 
-      raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
-
       if File.directory?(options[:require])
         require 'rails'
         if ::Rails::VERSION::MAJOR < 4
@@ -273,10 +289,7 @@ module Sidekiq
         end
         options[:tag] ||= default_tag
       else
-        not_required_message = "#{options[:require]} was not required, you should use an explicit path: " +
-            "./#{options[:require]} or /path/to/#{options[:require]}"
-
-        require(options[:require]) || raise(ArgumentError, not_required_message)
+        require options[:require]
       end
     end
 
@@ -292,8 +305,6 @@ module Sidekiq
     end
 
     def validate!
-      options[:queues] << 'default' if options[:queues].empty?
-
       if !File.exist?(options[:require]) ||
          (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
         logger.info "=================================================================="
@@ -319,6 +330,7 @@ module Sidekiq
 
         o.on '-d', '--daemon', "Daemonize process" do |arg|
           opts[:daemon] = arg
+          puts "WARNING: Daemonization mode will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services"
         end
 
         o.on '-e', '--environment ENV', "Application environment" do |arg|
@@ -358,10 +370,12 @@ module Sidekiq
 
         o.on '-L', '--logfile PATH', "path to writable logfile" do |arg|
           opts[:logfile] = arg
+          puts "WARNING: Logfile redirection will be removed in Sidekiq 6.0, see #4045. Sidekiq will only log to STDOUT"
         end
 
         o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
           opts[:pidfile] = arg
+          puts "WARNING: PID file creation will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services"
         end
 
         o.on '-V', '--version', "Print version and exit" do |arg|
@@ -375,11 +389,8 @@ module Sidekiq
         logger.info @parser
         die 1
       end
-      @parser.parse!(argv)
 
-      %w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename|
-        opts[:config_file] ||= filename if File.exist?(filename)
-      end
+      @parser.parse!(argv)
 
       opts
     end
@@ -399,23 +410,18 @@ module Sidekiq
       end
     end
 
-    def parse_config(cfile)
-      opts = {}
-      if File.exist?(cfile)
-        opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
-
-        if opts.respond_to? :deep_symbolize_keys!
-          opts.deep_symbolize_keys!
-        else
-          symbolize_keys_deep!(opts)
-        end
+    def parse_config(path)
+      opts = YAML.load(ERB.new(File.read(path)).result) || {}
 
-        opts = opts.merge(opts.delete(environment.to_sym) || {})
-        parse_queues(opts, opts.delete(:queues) || [])
+      if opts.respond_to? :deep_symbolize_keys!
+        opts.deep_symbolize_keys!
       else
-        # allow a non-existent config file so Sidekiq
-        # can be deployed by cap with just the defaults.
+        symbolize_keys_deep!(opts)
       end
+
+      opts = opts.merge(opts.delete(environment.to_sym) || {})
+      parse_queues(opts, opts.delete(:queues) || [])
+
       ns = opts.delete(:namespace)
       if ns
         # logger hasn't been initialized yet, puts is all we have.
@@ -429,10 +435,10 @@ module Sidekiq
       queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
     end
 
-    def parse_queue(opts, q, weight=nil)
-      [weight.to_i, 1].max.times do
-       (opts[:queues] ||= []) << q
-      end
+    def parse_queue(opts, queue, weight = nil)
+      opts[:queues] ||= []
+      raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
+      [weight.to_i, 1].max.times { opts[:queues] << queue }
       opts[:strict] = false if weight.to_i > 0
     end
   end

data/lib/sidekiq/client.rb

@@ -77,9 +77,10 @@ module Sidekiq
     end
 
     ##
-    # Push a large number of jobs to Redis.  In practice this method is only
-    # useful if you are pushing thousands of jobs or more.  This method
-    # cuts out the redis network round trip latency.
+    # Push a large number of jobs to Redis. This method cuts out the redis
+    # network round trip latency.  I wouldn't recommend pushing more than
+    # 1000 per call but YMMV based on network quality, size of job args, etc.
+    # A large number of jobs can cause a bit of Redis command processing latency.
     #
     # Takes the same arguments as #push except that args is expected to be
     # an Array of Arrays.  All other keys are duplicated for each job.  Each job