sidekiq 4.2.4 → 5.2.0

data/lib/sidekiq/cli.rb

@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-# encoding: utf-8
 $stdout.sync = true
 
 require 'yaml'
@@ -17,7 +16,7 @@ module Sidekiq
     include Singleton unless $TESTING
 
     PROCTITLES = [
-      proc { 'sidekiq'.freeze },
+      proc { 'sidekiq' },
       proc { Sidekiq::VERSION },
       proc { |me, data| data['tag'] },
       proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
@@ -43,6 +42,10 @@ module Sidekiq
       write_pid
     end
 
+    def jruby?
+      defined?(::JRUBY_VERSION)
+    end
+
     # Code within this method is not tested because it alters
     # global process state irreversibly.  PRs which improve the
     # test coverage of Sidekiq::CLI are welcomed.
@@ -51,11 +54,17 @@ module Sidekiq
       print_banner
 
       self_read, self_write = IO.pipe
+      sigs = %w(INT TERM TTIN TSTP)
+      # USR1 and USR2 don't work on the JVM
+      if !jruby?
+        sigs << 'USR1'
+        sigs << 'USR2'
+      end
 
-      %w(INT TERM USR1 USR2 TTIN).each do |sig|
+      sigs.each do |sig|
         begin
           trap sig do
-            self_write.puts(sig)
+            self_write.write("#{sig}\n")
           end
         rescue ArgumentError
           puts "Signal #{sig} not supported"
@@ -71,12 +80,21 @@ module Sidekiq
       ver = Sidekiq.redis_info['redis_version']
       raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
 
+      # Since the user can pass us a connection pool explicitly in the initializer, we
+      # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
+      cursize = Sidekiq.redis_pool.size
+      needed = Sidekiq.options[:concurrency] + 2
+      raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
+
+      # cache process identity
+      Sidekiq.options[:identity] = identity
+
       # Touch middleware so it isn't lazy loaded by multiple threads, #3043
       Sidekiq.server_middleware
 
       # Before this point, the process is initializing with just the main thread.
       # Starting here the process will now have multiple threads running.
-      fire_event(:startup)
+      fire_event(:startup, reverse: false, reraise: true)
 
       logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(', ')}" }
       logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}" }
@@ -122,33 +140,45 @@ module Sidekiq
 }
     end
 
-    def handle_signal(sig)
-      Sidekiq.logger.debug "Got #{sig} signal"
-      case sig
-      when 'INT'
-        # Handle Ctrl-C in JRuby like MRI
-        # http://jira.codehaus.org/browse/JRUBY-4637
-        raise Interrupt
-      when 'TERM'
-        # Heroku sends TERM and then waits 10 seconds for process to exit.
-        raise Interrupt
-      when 'USR1'
+    SIGNAL_HANDLERS = {
+      # Ctrl-C in terminal
+      'INT' => ->(cli) { raise Interrupt },
+      # TERM is the signal that Sidekiq must exit.
+      # Heroku sends TERM and then waits 30 seconds for process to exit.
+      'TERM' => ->(cli) { raise Interrupt },
+      'USR1' => ->(cli) {
         Sidekiq.logger.info "Received USR1, no longer accepting new work"
-        launcher.quiet
-      when 'USR2'
+        cli.launcher.quiet
+      },
+      'TSTP' => ->(cli) {
+        Sidekiq.logger.info "Received TSTP, no longer accepting new work"
+        cli.launcher.quiet
+      },
+      'USR2' => ->(cli) {
         if Sidekiq.options[:logfile]
           Sidekiq.logger.info "Received USR2, reopening log file"
           Sidekiq::Logging.reopen_logs
         end
-      when 'TTIN'
+      },
+      'TTIN' => ->(cli) {
         Thread.list.each do |thread|
-          Sidekiq.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
+          Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['sidekiq_label']}"
           if thread.backtrace
             Sidekiq.logger.warn thread.backtrace.join("\n")
           else
             Sidekiq.logger.warn "<no backtrace available>"
           end
         end
+      },
+    }
+
+    def handle_signal(sig)
+      Sidekiq.logger.debug "Got #{sig} signal"
+      handy = SIGNAL_HANDLERS[sig]
+      if handy
+        handy.call(self)
+      else
+        Sidekiq.logger.info { "No signal handler for #{sig}" }
       end
     end
 
@@ -197,6 +227,14 @@ module Sidekiq
       @environment = cli_env || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
     end
 
+    def symbolize_keys_deep!(hash)
+      hash.keys.each do |k|
+        symkey = k.respond_to?(:to_sym) ? k.to_sym : k
+        hash[symkey] = hash.delete k
+        symbolize_keys_deep! hash[symkey] if hash[symkey].kind_of? Hash
+      end
+    end
+
     alias_method :die, :exit
     alias_method :☠, :exit
 
@@ -209,7 +247,6 @@ module Sidekiq
 
       opts[:strict] = true if opts[:strict].nil?
       opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"]
-      opts[:identity] = identity
 
       options.merge!(opts)
     end
@@ -226,11 +263,10 @@ module Sidekiq
       if File.directory?(options[:require])
         require 'rails'
         if ::Rails::VERSION::MAJOR < 4
-          require 'sidekiq/rails'
-          require File.expand_path("#{options[:require]}/config/environment.rb")
-          ::Rails.application.eager_load!
+          raise "Sidekiq no longer supports this version of Rails"
         elsif ::Rails::VERSION::MAJOR == 4
           # Painful contortions, see 1791 for discussion
+          # No autoloading, we want to force eager load for everything.
           require File.expand_path("#{options[:require]}/config/application.rb")
           ::Rails::Application.initializer "sidekiq.eager_load" do
             ::Rails.application.config.eager_load = true
@@ -267,7 +303,7 @@ module Sidekiq
       if !File.exist?(options[:require]) ||
          (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
         logger.info "=================================================================="
-        logger.info "  Please point sidekiq to a Rails 3/4 application or a Ruby file  "
+        logger.info "  Please point sidekiq to a Rails 4/5 application or a Ruby file  "
         logger.info "  to load your worker classes with -r [DIR|FILE]."
         logger.info "=================================================================="
         logger.info @parser
@@ -299,6 +335,8 @@ module Sidekiq
           opts[:tag] = arg
         end
 
+        # this index remains here for backwards compatibility but none of the Sidekiq
+        # family use this value anymore.  it was used by Pro's original reliable_fetch.
         o.on '-i', '--index INT', "unique process index on this machine" do |arg|
           opts[:index] = Integer(arg.match(/\d+/)[0])
         end
@@ -371,7 +409,14 @@ module Sidekiq
       opts = {}
       if File.exist?(cfile)
         opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
-        opts = opts.merge(opts.delete(environment) || {})
+
+        if opts.respond_to? :deep_symbolize_keys!
+          opts.deep_symbolize_keys!
+        else
+          symbolize_keys_deep!(opts)
+        end
+
+        opts = opts.merge(opts.delete(environment.to_sym) || {})
         parse_queues(opts, opts.delete(:queues) || [])
       else
         # allow a non-existent config file so Sidekiq

data/lib/sidekiq/client.rb

@@ -48,9 +48,15 @@ module Sidekiq
     #   queue - the named queue to use, default 'default'
     #   class - the worker class to call, required
     #   args - an array of simple arguments to the perform method, must be JSON-serializable
+    #   at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
     #   retry - whether to retry this job if it fails, default true or an integer number of retries
     #   backtrace - whether to save any error backtrace, default false
     #
+    # If class is set to the class name, the jobs' options will be based on Sidekiq's default
+    # worker options. Otherwise, they will be based on the job class's options.
+    #
+    # Any options valid for a worker class's sidekiq_options are also available here.
+    #
     # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
     # symbols in 'args' will be converted to strings.  Note that +backtrace: true+ can take quite a bit of
     # space in Redis; a large volume of failing jobs can start Redis swapping if you aren't careful.
@@ -71,9 +77,10 @@ module Sidekiq
     end
 
     ##
-    # Push a large number of jobs to Redis.  In practice this method is only
-    # useful if you are pushing thousands of jobs or more.  This method
-    # cuts out the redis network round trip latency.
+    # Push a large number of jobs to Redis. This method cuts out the redis
+    # network round trip latency.  I wouldn't recommend pushing more than
+    # 1000 per call but YMMV based on network quality, size of job args, etc.
+    # A large number of jobs can cause a bit of Redis command processing latency.
     #
     # Takes the same arguments as #push except that args is expected to be
     # an Array of Arrays.  All other keys are duplicated for each job.  Each job
@@ -113,11 +120,10 @@ module Sidekiq
     def self.via(pool)
       raise ArgumentError, "No pool given" if pool.nil?
       current_sidekiq_pool = Thread.current[:sidekiq_via_pool]
-      raise RuntimeError, "Sidekiq::Client.via is not re-entrant" if current_sidekiq_pool && current_sidekiq_pool != pool
       Thread.current[:sidekiq_via_pool] = pool
       yield
     ensure
-      Thread.current[:sidekiq_via_pool] = nil
+      Thread.current[:sidekiq_via_pool] = current_sidekiq_pool
     end
 
     class << self
@@ -158,7 +164,7 @@ module Sidekiq
         ts = (int < 1_000_000_000 ? now + int : int)
 
         item = { 'class' => klass, 'args' => args, 'at' => ts, 'queue' => queue }
-        item.delete('at'.freeze) if ts <= now
+        item.delete('at') if ts <= now
 
         klass.client_push(item)
       end
@@ -184,18 +190,18 @@ module Sidekiq
 
     def atomic_push(conn, payloads)
       if payloads.first['at']
-        conn.zadd('schedule'.freeze, payloads.map do |hash|
-          at = hash.delete('at'.freeze).to_s
+        conn.zadd('schedule', payloads.map do |hash|
+          at = hash.delete('at').to_s
           [at, Sidekiq.dump_json(hash)]
         end)
       else
         q = payloads.first['queue']
         now = Time.now.to_f
         to_push = payloads.map do |entry|
-          entry['enqueued_at'.freeze] = now
+          entry['enqueued_at'] = now
           Sidekiq.dump_json(entry)
         end
-        conn.sadd('queues'.freeze, q)
+        conn.sadd('queues', q)
         conn.lpush("queue:#{q}", to_push)
       end
     end
@@ -209,24 +215,25 @@ module Sidekiq
     end
 
     def normalize_item(item)
-      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class'.freeze) && item.has_key?('args'.freeze)
+      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class') && item.has_key?('args')
       raise(ArgumentError, "Job args must be an Array") unless item['args'].is_a?(Array)
-      raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'.freeze].is_a?(Class) || item['class'.freeze].is_a?(String)
+      raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'].is_a?(Class) || item['class'].is_a?(String)
+      raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.has_key?('at') && !item['at'].is_a?(Numeric)
       #raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
 
-      normalized_hash(item['class'.freeze])
+      normalized_hash(item['class'])
         .each{ |key, value| item[key] = value if item[key].nil? }
 
-      item['class'.freeze] = item['class'.freeze].to_s
-      item['queue'.freeze] = item['queue'.freeze].to_s
-      item['jid'.freeze] ||= SecureRandom.hex(12)
-      item['created_at'.freeze] ||= Time.now.to_f
+      item['class'] = item['class'].to_s
+      item['queue'] = item['queue'].to_s
+      item['jid'] ||= SecureRandom.hex(12)
+      item['created_at'] ||= Time.now.to_f
       item
     end
 
     def normalized_hash(item_class)
       if item_class.is_a?(Class)
-        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") if !item_class.respond_to?('get_sidekiq_options'.freeze)
+        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") if !item_class.respond_to?('get_sidekiq_options')
         item_class.get_sidekiq_options
       else
         Sidekiq.default_worker_options

data/lib/sidekiq/core_ext.rb

@@ -1,106 +1 @@
-# frozen_string_literal: true
-begin
-  require 'active_support/core_ext/class/attribute'
-rescue LoadError
-
-  # A dumbed down version of ActiveSupport's
-  # Class#class_attribute helper.
-  class Class
-    def class_attribute(*attrs)
-      instance_writer = true
-
-      attrs.each do |name|
-        class_eval <<-RUBY, __FILE__, __LINE__ + 1
-          def self.#{name}() nil end
-          def self.#{name}?() !!#{name} end
-
-          def self.#{name}=(val)
-            singleton_class.class_eval do
-              define_method(:#{name}) { val }
-            end
-
-            if singleton_class?
-              class_eval do
-                def #{name}
-                  defined?(@#{name}) ? @#{name} : singleton_class.#{name}
-                end
-              end
-            end
-            val
-          end
-
-          def #{name}
-            defined?(@#{name}) ? @#{name} : self.class.#{name}
-          end
-
-          def #{name}?
-            !!#{name}
-          end
-        RUBY
-
-        attr_writer name if instance_writer
-      end
-    end
-
-    private
-    def singleton_class?
-      ancestors.first != self
-    end
-  end
-end
-
-begin
-  require 'active_support/core_ext/hash/keys'
-  require 'active_support/core_ext/hash/deep_merge'
-rescue LoadError
-  class Hash
-    def stringify_keys
-      keys.each do |key|
-        self[key.to_s] = delete(key)
-      end
-      self
-    end if !{}.respond_to?(:stringify_keys)
-
-    def symbolize_keys
-      keys.each do |key|
-        self[(key.to_sym rescue key) || key] = delete(key)
-      end
-      self
-    end if !{}.respond_to?(:symbolize_keys)
-
-    def deep_merge(other_hash, &block)
-      dup.deep_merge!(other_hash, &block)
-    end if !{}.respond_to?(:deep_merge)
-
-    def deep_merge!(other_hash, &block)
-      other_hash.each_pair do |k,v|
-        tv = self[k]
-        if tv.is_a?(Hash) && v.is_a?(Hash)
-          self[k] = tv.deep_merge(v, &block)
-        else
-          self[k] = block && tv ? block.call(k, tv, v) : v
-        end
-      end
-      self
-    end if !{}.respond_to?(:deep_merge!)
-  end
-end
-
-begin
-  require 'active_support/core_ext/string/inflections'
-rescue LoadError
-  class String
-    def constantize
-      names = self.split('::')
-      names.shift if names.empty? || names.first.empty?
-
-      constant = Object
-      names.each do |name|
-        constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
-      end
-      constant
-    end
-  end if !"".respond_to?(:constantize)
-end
-
-
+raise "no longer used, will be removed in 5.1"

data/lib/sidekiq/delay.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+module Sidekiq
+  module Extensions
+
+    def self.enable_delay!
+      if defined?(::ActiveSupport)
+        require 'sidekiq/extensions/active_record'
+        require 'sidekiq/extensions/action_mailer'
+
+        # Need to patch Psych so it can autoload classes whose names are serialized
+        # in the delayed YAML.
+        Psych::Visitors::ToRuby.prepend(Sidekiq::Extensions::PsychAutoload)
+
+        ActiveSupport.on_load(:active_record) do
+          include Sidekiq::Extensions::ActiveRecord
+        end
+        ActiveSupport.on_load(:action_mailer) do
+          extend Sidekiq::Extensions::ActionMailer
+        end
+      end
+
+      require 'sidekiq/extensions/class_methods'
+      Module.__send__(:include, Sidekiq::Extensions::Klass)
+    end
+
+    module PsychAutoload
+      def resolve_class(klass_name)
+        return nil if !klass_name || klass_name.empty?
+        # constantize
+        names = klass_name.split('::')
+        names.shift if names.empty? || names.first.empty?
+
+        names.inject(Object) do |constant, name|
+          constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+        end
+      rescue NameError
+        super
+      end
+    end
+  end
+end
+

data/lib/sidekiq/exception_handler.rb

@@ -7,11 +7,10 @@ module Sidekiq
     class Logger
       def call(ex, ctxHash)
         Sidekiq.logger.warn(Sidekiq.dump_json(ctxHash)) if !ctxHash.empty?
-        Sidekiq.logger.warn "#{ex.class.name}: #{ex.message}"
-        Sidekiq.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
+        Sidekiq.logger.warn("#{ex.class.name}: #{ex.message}")
+        Sidekiq.logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
       end
 
-      # Set up default handler which just logs the error
       Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
     end
 
@@ -26,6 +25,5 @@ module Sidekiq
         end
       end
     end
-
   end
 end

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -3,6 +3,8 @@ require 'yaml'
 
 module Sidekiq
   module Extensions
+    SIZE_LIMIT = 8_192
+
     class Proxy < BasicObject
       def initialize(performable, target, options={})
         @performable = performable
@@ -17,7 +19,11 @@ module Sidekiq
         # to JSON and then deserialized on the other side back into a
         # Ruby object.
         obj = [@target, name, args]
-        @performable.client_push({ 'class' => @performable, 'args' => [::YAML.dump(obj)] }.merge(@opts))
+        marshalled = ::YAML.dump(obj)
+        if marshalled.size > SIZE_LIMIT
+          ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
+        end
+        @performable.client_push({ 'class' => @performable, 'args' => [marshalled] }.merge(@opts))
       end
     end
 

data/lib/sidekiq/fetch.rb

@@ -13,7 +13,7 @@ module Sidekiq
       end
 
       def queue_name
-        queue.sub(/.*queue:/, ''.freeze)
+        queue.sub(/.*queue:/, '')
       end
 
       def requeue

data/lib/sidekiq/job_logger.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+module Sidekiq
+  class JobLogger
+
+    def call(item, queue)
+      start = Time.now
+      logger.info("start")
+      yield
+      logger.info("done: #{elapsed(start)} sec")
+    rescue Exception
+      logger.info("fail: #{elapsed(start)} sec")
+      raise
+    end
+
+    private
+
+    def elapsed(start)
+      (Time.now - start).round(3)
+    end
+
+    def logger
+      Sidekiq.logger
+    end
+  end
+end