sidekiq 3.5.4 → 5.2.7

data/lib/sidekiq/cli.rb

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# frozen_string_literal: true
 $stdout.sync = true
 
 require 'yaml'
@@ -9,53 +9,55 @@ require 'fileutils'
 
 require 'sidekiq'
 require 'sidekiq/util'
+require 'sidekiq/launcher'
 
 module Sidekiq
-  # We are shutting down Sidekiq but what about workers that
-  # are working on some long job?  This error is
-  # raised in workers that have not finished within the hard
-  # timeout limit.  This is needed to rollback db transactions,
-  # otherwise Ruby's Thread#kill will commit.  See #377.
-  # DO NOT RESCUE THIS ERROR.
-  class Shutdown < Interrupt; end
-
   class CLI
     include Util
     include Singleton unless $TESTING
 
-    # Used for CLI testing
-    attr_accessor :code
+    PROCTITLES = [
+      proc { 'sidekiq' },
+      proc { Sidekiq::VERSION },
+      proc { |me, data| data['tag'] },
+      proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
+      proc { |me, data| "stopping" if me.stopping? },
+    ]
+
     attr_accessor :launcher
     attr_accessor :environment
 
-    def initialize
-      @code = nil
-    end
-
-    def parse(args=ARGV)
-      @code = nil
-
+    def parse(args = ARGV)
       setup_options(args)
       initialize_logger
       validate!
-      daemonize
-      write_pid
-      load_celluloid
+    end
+
+    def jruby?
+      defined?(::JRUBY_VERSION)
     end
 
     # Code within this method is not tested because it alters
     # global process state irreversibly.  PRs which improve the
     # test coverage of Sidekiq::CLI are welcomed.
     def run
+      daemonize if options[:daemon]
+      write_pid
       boot_system
-      print_banner
+      print_banner if environment == 'development' && $stdout.tty?
 
       self_read, self_write = IO.pipe
+      sigs = %w(INT TERM TTIN TSTP)
+      # USR1 and USR2 don't work on the JVM
+      if !jruby?
+        sigs << 'USR1'
+        sigs << 'USR2'
+      end
 
-      %w(INT TERM USR1 USR2 TTIN).each do |sig|
+      sigs.each do |sig|
         begin
           trap sig do
-            self_write.puts(sig)
+            self_write.write("#{sig}\n")
           end
         rescue ArgumentError
           puts "Signal #{sig} not supported"
@@ -66,22 +68,39 @@ module Sidekiq
       logger.info Sidekiq::LICENSE
       logger.info "Upgrade to Sidekiq Pro for more features and support: http://sidekiq.org" unless defined?(::Sidekiq::Pro)
 
-      fire_event(:startup)
+      # touch the connection pool so it is created before we
+      # fire startup and start multithreading.
+      ver = Sidekiq.redis_info['redis_version']
+      raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
+      logger.warn "Sidekiq 6.0 will require Redis 4.0+, you are using Redis v#{ver}" if ver < '4'
 
-      logger.debug {
-        "Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}"
-      }
+      # Since the user can pass us a connection pool explicitly in the initializer, we
+      # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
+      cursize = Sidekiq.redis_pool.size
+      needed = Sidekiq.options[:concurrency] + 2
+      raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
 
-      Sidekiq.redis do |conn|
-        # touch the connection pool so it is created before we
-        # launch the actors.
-      end
+      # cache process identity
+      Sidekiq.options[:identity] = identity
+
+      # Touch middleware so it isn't lazy loaded by multiple threads, #3043
+      Sidekiq.server_middleware
 
+      # Before this point, the process is initializing with just the main thread.
+      # Starting here the process will now have multiple threads running.
+      fire_event(:startup, reverse: false, reraise: true)
+
+      logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(', ')}" }
+      logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}" }
+
+      launch(self_read)
+    end
+
+    def launch(self_read)
       if !options[:daemon]
         logger.info 'Starting processing, hit Ctrl-C to stop'
       end
 
-      require 'sidekiq/launcher'
       @launcher = Sidekiq::Launcher.new(options)
 
       begin
@@ -96,6 +115,7 @@ module Sidekiq
         launcher.stop
         # Explicitly exit so busy Processor threads can't block
         # process shutdown.
+        logger.info "Bye!"
         exit(0)
       end
     end
@@ -117,70 +137,60 @@ module Sidekiq
 }
     end
 
-    def handle_signal(sig)
-      Sidekiq.logger.debug "Got #{sig} signal"
-      case sig
-      when 'INT'
-        # Handle Ctrl-C in JRuby like MRI
-        # http://jira.codehaus.org/browse/JRUBY-4637
-        raise Interrupt
-      when 'TERM'
-        # Heroku sends TERM and then waits 10 seconds for process to exit.
-        raise Interrupt
-      when 'USR1'
+    SIGNAL_HANDLERS = {
+      # Ctrl-C in terminal
+      'INT' => ->(cli) { raise Interrupt },
+      # TERM is the signal that Sidekiq must exit.
+      # Heroku sends TERM and then waits 30 seconds for process to exit.
+      'TERM' => ->(cli) { raise Interrupt },
+      'USR1' => ->(cli) {
         Sidekiq.logger.info "Received USR1, no longer accepting new work"
-        launcher.manager.async.stop
-        fire_event(:quiet, true)
-      when 'USR2'
+        cli.launcher.quiet
+      },
+      'TSTP' => ->(cli) {
+        Sidekiq.logger.info "Received TSTP, no longer accepting new work"
+        cli.launcher.quiet
+      },
+      'USR2' => ->(cli) {
         if Sidekiq.options[:logfile]
           Sidekiq.logger.info "Received USR2, reopening log file"
           Sidekiq::Logging.reopen_logs
         end
-      when 'TTIN'
+      },
+      'TTIN' => ->(cli) {
         Thread.list.each do |thread|
-          Sidekiq.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
+          Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['sidekiq_label']}"
           if thread.backtrace
             Sidekiq.logger.warn thread.backtrace.join("\n")
           else
             Sidekiq.logger.warn "<no backtrace available>"
           end
         end
+      },
+    }
+
+    def handle_signal(sig)
+      Sidekiq.logger.debug "Got #{sig} signal"
+      handy = SIGNAL_HANDLERS[sig]
+      if handy
+        handy.call(self)
+      else
+        Sidekiq.logger.info { "No signal handler for #{sig}" }
       end
     end
 
     private
 
     def print_banner
-      # Print logo and banner for development
-      if environment == 'development' && $stdout.tty?
-        puts "\e[#{31}m"
-        puts Sidekiq::CLI.banner
-        puts "\e[0m"
-      end
-    end
-
-    def load_celluloid
-      raise "Celluloid cannot be required until here, or it will break Sidekiq's daemonization" if defined?(::Celluloid) && options[:daemon]
-
-      # Celluloid can't be loaded until after we've daemonized
-      # because it spins up threads and creates locks which get
-      # into a very bad state if forked.
-      require 'celluloid/current'
-      Celluloid.logger = (options[:verbose] ? Sidekiq.logger : nil)
-
-      require 'sidekiq/manager'
-      require 'sidekiq/scheduled'
+      puts "\e[#{31}m"
+      puts Sidekiq::CLI.banner
+      puts "\e[0m"
     end
 
     def daemonize
-      return unless options[:daemon]
-
       raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
-      files_to_reopen = []
-      ObjectSpace.each_object(File) do |file|
-        files_to_reopen << file unless file.closed?
-      end
 
+      files_to_reopen = ObjectSpace.each_object(File).reject { |f| f.closed? }
       ::Process.daemon(true, true)
 
       files_to_reopen.each do |file|
@@ -206,18 +216,50 @@ module Sidekiq
       @environment = cli_env || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
     end
 
+    def symbolize_keys_deep!(hash)
+      hash.keys.each do |k|
+        symkey = k.respond_to?(:to_sym) ? k.to_sym : k
+        hash[symkey] = hash.delete k
+        symbolize_keys_deep! hash[symkey] if hash[symkey].kind_of? Hash
+      end
+    end
+
     alias_method :die, :exit
     alias_method :☠, :exit
 
     def setup_options(args)
+      # parse CLI options
       opts = parse_options(args)
+
       set_environment opts[:environment]
 
-      cfile = opts[:config_file]
-      opts = parse_config(cfile).merge(opts) if cfile
+      # check config file presence
+      if opts[:config_file]
+        if opts[:config_file] && !File.exist?(opts[:config_file])
+          raise ArgumentError, "No such file #{opts[:config_file]}"
+        end
+      else
+        config_dir = if File.directory?(opts[:require].to_s)
+          File.join(opts[:require], 'config')
+        else
+          File.join(options[:require], 'config')
+        end
+
+        %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
+          path = File.join(config_dir, config_file)
+          opts[:config_file] ||= path if File.exist?(path)
+        end
+      end
 
+      # parse config file options
+      opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
+
+      # set defaults
+      opts[:queues] = Array(opts[:queues]) << 'default' if opts[:queues].nil? || opts[:queues].empty?
       opts[:strict] = true if opts[:strict].nil?
+      opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
 
+      # merge with defaults
       options.merge!(opts)
     end
 
@@ -228,22 +270,22 @@ module Sidekiq
     def boot_system
       ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
 
-      raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
-
       if File.directory?(options[:require])
         require 'rails'
         if ::Rails::VERSION::MAJOR < 4
-          require 'sidekiq/rails'
-          require File.expand_path("#{options[:require]}/config/environment.rb")
-          ::Rails.application.eager_load!
-        else
+          raise "Sidekiq no longer supports this version of Rails"
+        elsif ::Rails::VERSION::MAJOR == 4
           # Painful contortions, see 1791 for discussion
+          # No autoloading, we want to force eager load for everything.
           require File.expand_path("#{options[:require]}/config/application.rb")
           ::Rails::Application.initializer "sidekiq.eager_load" do
             ::Rails.application.config.eager_load = true
           end
           require 'sidekiq/rails'
           require File.expand_path("#{options[:require]}/config/environment.rb")
+        else
+          require 'sidekiq/rails'
+          require File.expand_path("#{options[:require]}/config/environment.rb")
         end
         options[:tag] ||= default_tag
       else
@@ -263,12 +305,10 @@ module Sidekiq
     end
 
     def validate!
-      options[:queues] << 'default' if options[:queues].empty?
-
       if !File.exist?(options[:require]) ||
          (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
         logger.info "=================================================================="
-        logger.info "  Please point sidekiq to a Rails 3/4 application or a Ruby file  "
+        logger.info "  Please point sidekiq to a Rails 4/5 application or a Ruby file  "
         logger.info "  to load your worker classes with -r [DIR|FILE]."
         logger.info "=================================================================="
         logger.info @parser
@@ -290,6 +330,7 @@ module Sidekiq
 
         o.on '-d', '--daemon', "Daemonize process" do |arg|
           opts[:daemon] = arg
+          puts "WARNING: Daemonization mode will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services"
         end
 
         o.on '-e', '--environment ENV', "Application environment" do |arg|
@@ -300,6 +341,8 @@ module Sidekiq
           opts[:tag] = arg
         end
 
+        # this index remains here for backwards compatibility but none of the Sidekiq
+        # family use this value anymore.  it was used by Pro's original reliable_fetch.
         o.on '-i', '--index INT', "unique process index on this machine" do |arg|
           opts[:index] = Integer(arg.match(/\d+/)[0])
         end
@@ -327,10 +370,12 @@ module Sidekiq
 
         o.on '-L', '--logfile PATH', "path to writable logfile" do |arg|
           opts[:logfile] = arg
+          puts "WARNING: Logfile redirection will be removed in Sidekiq 6.0, see #4045. Sidekiq will only log to STDOUT"
         end
 
         o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
           opts[:pidfile] = arg
+          puts "WARNING: PID file creation will be removed in Sidekiq 6.0, see #4045. Please use a proper process supervisor to start and manage your services"
         end
 
         o.on '-V', '--version', "Print version and exit" do |arg|
@@ -344,11 +389,8 @@ module Sidekiq
         logger.info @parser
         die 1
       end
-      @parser.parse!(argv)
 
-      %w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename|
-        opts[:config_file] ||= filename if File.exist?(filename)
-      end
+      @parser.parse!(argv)
 
       opts
     end
@@ -368,16 +410,18 @@ module Sidekiq
       end
     end
 
-    def parse_config(cfile)
-      opts = {}
-      if File.exist?(cfile)
-        opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
-        opts = opts.merge(opts.delete(environment) || {})
-        parse_queues(opts, opts.delete(:queues) || [])
+    def parse_config(path)
+      opts = YAML.load(ERB.new(File.read(path)).result) || {}
+
+      if opts.respond_to? :deep_symbolize_keys!
+        opts.deep_symbolize_keys!
       else
-        # allow a non-existent config file so Sidekiq
-        # can be deployed by cap with just the defaults.
+        symbolize_keys_deep!(opts)
       end
+
+      opts = opts.merge(opts.delete(environment.to_sym) || {})
+      parse_queues(opts, opts.delete(:queues) || [])
+
       ns = opts.delete(:namespace)
       if ns
         # logger hasn't been initialized yet, puts is all we have.
@@ -391,10 +435,10 @@ module Sidekiq
       queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
     end
 
-    def parse_queue(opts, q, weight=nil)
-      [weight.to_i, 1].max.times do
-       (opts[:queues] ||= []) << q
-      end
+    def parse_queue(opts, queue, weight = nil)
+      opts[:queues] ||= []
+      raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
+      [weight.to_i, 1].max.times { opts[:queues] << queue }
       opts[:strict] = false if weight.to_i > 0
     end
   end

data/lib/sidekiq/client.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'securerandom'
 require 'sidekiq/middleware/chain'
 
@@ -35,10 +36,8 @@ module Sidekiq
     #   Sidekiq::Client.new(ConnectionPool.new { Redis.new })
     #
     # Generally this is only needed for very large Sidekiq installs processing
-    # more than thousands jobs per second.  I do not recommend sharding unless
-    # you truly cannot scale any other way (e.g. splitting your app into smaller apps).
-    # Some features, like the API, do not support sharding: they are designed to work
-    # against a single Redis instance only.
+    # thousands of jobs per second.  I don't recommend sharding unless you
+    # cannot scale any other way (e.g. splitting your app into smaller apps).
     def initialize(redis_pool=nil)
       @redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
     end
@@ -49,11 +48,18 @@ module Sidekiq
     #   queue - the named queue to use, default 'default'
     #   class - the worker class to call, required
     #   args - an array of simple arguments to the perform method, must be JSON-serializable
-    #   retry - whether to retry this job if it fails, true or false, default true
+    #   at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
+    #   retry - whether to retry this job if it fails, default true or an integer number of retries
     #   backtrace - whether to save any error backtrace, default false
     #
+    # If class is set to the class name, the jobs' options will be based on Sidekiq's default
+    # worker options. Otherwise, they will be based on the job class's options.
+    #
+    # Any options valid for a worker class's sidekiq_options are also available here.
+    #
     # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
-    # symbols in 'args' will be converted to strings.
+    # symbols in 'args' will be converted to strings.  Note that +backtrace: true+ can take quite a bit of
+    # space in Redis; a large volume of failing jobs can start Redis swapping if you aren't careful.
     #
     # Returns a unique Job ID.  If middleware stops the job, nil will be returned instead.
     #
@@ -71,10 +77,10 @@ module Sidekiq
     end
 
     ##
-    # Push a large number of jobs to Redis.  In practice this method is only
-    # useful if you are pushing tens of thousands of jobs or more, or if you need
-    # to ensure that a batch doesn't complete prematurely.  This method
-    # basically cuts down on the redis round trip latency.
+    # Push a large number of jobs to Redis. This method cuts out the redis
+    # network round trip latency.  I wouldn't recommend pushing more than
+    # 1000 per call but YMMV based on network quality, size of job args, etc.
+    # A large number of jobs can cause a bit of Redis command processing latency.
     #
     # Takes the same arguments as #push except that args is expected to be
     # an Array of Arrays.  All other keys are duplicated for each job.  Each job
@@ -84,10 +90,15 @@ module Sidekiq
     # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
     # than the number given if the middleware stopped processing for one or more jobs.
     def push_bulk(items)
+      arg = items['args'].first
+      return [] unless arg # no jobs to push
+      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !arg.is_a?(Array)
+
       normed = normalize_item(items)
       payloads = items['args'].map do |args|
-        raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !args.is_a?(Array)
-        process_single(items['class'], normed.merge('args' => args, 'jid' => SecureRandom.hex(12), 'enqueued_at' => Time.now.to_f))
+        copy = normed.merge('args' => args, 'jid' => SecureRandom.hex(12), 'enqueued_at' => Time.now.to_f)
+        result = process_single(items['class'], copy)
+        result ? result : nil
       end.compact
 
       raw_push(payloads) if !payloads.empty?
@@ -104,26 +115,19 @@ module Sidekiq
     #   end
     #
     # Generally this is only needed for very large Sidekiq installs processing
-    # more than thousands jobs per second.  I do not recommend sharding unless
-    # you truly cannot scale any other way (e.g. splitting your app into smaller apps).
-    # Some features, like the API, do not support sharding: they are designed to work
-    # against a single Redis instance.
+    # thousands of jobs per second.  I do not recommend sharding unless
+    # you cannot scale any other way (e.g. splitting your app into smaller apps).
     def self.via(pool)
       raise ArgumentError, "No pool given" if pool.nil?
-      raise RuntimeError, "Sidekiq::Client.via is not re-entrant" if x = Thread.current[:sidekiq_via_pool] && x != pool
+      current_sidekiq_pool = Thread.current[:sidekiq_via_pool]
       Thread.current[:sidekiq_via_pool] = pool
       yield
     ensure
-      Thread.current[:sidekiq_via_pool] = nil
+      Thread.current[:sidekiq_via_pool] = current_sidekiq_pool
     end
 
     class << self
 
-      # deprecated
-      def default
-        @default ||= new
-      end
-
       def push(item)
         new.push(item)
       end
@@ -160,7 +164,7 @@ module Sidekiq
         ts = (int < 1_000_000_000 ? now + int : int)
 
         item = { 'class' => klass, 'args' => args, 'at' => ts, 'queue' => queue }
-        item.delete('at'.freeze) if ts <= now
+        item.delete('at') if ts <= now
 
         klass.client_push(item)
       end
@@ -186,18 +190,18 @@ module Sidekiq
 
     def atomic_push(conn, payloads)
       if payloads.first['at']
-        conn.zadd('schedule'.freeze, payloads.map do |hash|
-          at = hash.delete('at'.freeze).to_s
+        conn.zadd('schedule', payloads.map do |hash|
+          at = hash.delete('at').to_s
           [at, Sidekiq.dump_json(hash)]
         end)
       else
         q = payloads.first['queue']
         now = Time.now.to_f
         to_push = payloads.map do |entry|
-          entry['enqueued_at'.freeze] = now
+          entry['enqueued_at'] = now
           Sidekiq.dump_json(entry)
         end
-        conn.sadd('queues'.freeze, q)
+        conn.sadd('queues', q)
         conn.lpush("queue:#{q}", to_push)
       end
     end
@@ -211,23 +215,25 @@ module Sidekiq
     end
 
     def normalize_item(item)
-      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class'.freeze) && item.has_key?('args'.freeze)
+      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class') && item.has_key?('args')
       raise(ArgumentError, "Job args must be an Array") unless item['args'].is_a?(Array)
-      raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'.freeze].is_a?(Class) || item['class'.freeze].is_a?(String)
+      raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'].is_a?(Class) || item['class'].is_a?(String)
+      raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.has_key?('at') && !item['at'].is_a?(Numeric)
+      #raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
 
-      normalized_hash(item['class'.freeze])
+      normalized_hash(item['class'])
         .each{ |key, value| item[key] = value if item[key].nil? }
 
-      item['class'.freeze] = item['class'.freeze].to_s
-      item['queue'.freeze] = item['queue'.freeze].to_s
-      item['jid'.freeze] ||= SecureRandom.hex(12)
-      item['created_at'.freeze] ||= Time.now.to_f
+      item['class'] = item['class'].to_s
+      item['queue'] = item['queue'].to_s
+      item['jid'] ||= SecureRandom.hex(12)
+      item['created_at'] ||= Time.now.to_f
       item
     end
 
     def normalized_hash(item_class)
       if item_class.is_a?(Class)
-        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") if !item_class.respond_to?('get_sidekiq_options'.freeze)
+        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") if !item_class.respond_to?('get_sidekiq_options')
         item_class.get_sidekiq_options
       else
         Sidekiq.default_worker_options