sidekiq 6.0.7 → 6.4.0

data/lib/sidekiq/cli.rb

@@ -33,8 +33,9 @@ module Sidekiq
     # Code within this method is not tested because it alters
     # global process state irreversibly.  PRs which improve the
     # test coverage of Sidekiq::CLI are welcomed.
-    def run
-      boot_system
+    def run(boot_app: true)
+      boot_application if boot_app
+
       if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
         print_banner
       end
@@ -43,9 +44,17 @@ module Sidekiq
       self_read, self_write = IO.pipe
       sigs = %w[INT TERM TTIN TSTP]
       # USR1 and USR2 don't work on the JVM
-      sigs << "USR2" unless jruby?
+      sigs << "USR2" if Sidekiq.pro? && !jruby?
       sigs.each do |sig|
-        trap sig do
+        old_handler = Signal.trap(sig) do
+          if old_handler.respond_to?(:call)
+            begin
+              old_handler.call
+            rescue Exception => exc
+              # signal handlers can't use Logger so puts only
+              puts ["Error in #{sig} handler", exc].inspect
+            end
+          end
           self_write.puts(sig)
         end
       rescue ArgumentError
@@ -58,9 +67,22 @@ module Sidekiq
 
       # touch the connection pool so it is created before we
       # fire startup and start multithreading.
-      ver = Sidekiq.redis_info["redis_version"]
+      info = Sidekiq.redis_info
+      ver = info["redis_version"]
       raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
 
+      maxmemory_policy = info["maxmemory_policy"]
+      if maxmemory_policy != "noeviction"
+        logger.warn <<~EOM
+
+
+          WARNING: Your Redis instance will evict Sidekiq data under heavy load.
+          The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
+          See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
+
+        EOM
+      end
+
       # Since the user can pass us a connection pool explicitly in the initializer, we
       # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
       cursize = Sidekiq.redis_pool.size
@@ -228,8 +250,7 @@ module Sidekiq
       opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
 
       # set defaults
-      opts[:queues] = ["default"] if opts[:queues].nil? || opts[:queues].empty?
-      opts[:strict] = true if opts[:strict].nil?
+      opts[:queues] = ["default"] if opts[:queues].nil?
       opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
 
       # merge with defaults
@@ -240,7 +261,7 @@ module Sidekiq
       Sidekiq.options
     end
 
-    def boot_system
+    def boot_application
       ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
 
       if File.directory?(options[:require])
@@ -359,7 +380,9 @@ module Sidekiq
     end
 
     def parse_config(path)
-      opts = YAML.load(ERB.new(File.read(path)).result) || {}
+      erb = ERB.new(File.read(path))
+      erb.filename = File.expand_path(path)
+      opts = YAML.load(erb.result) || {}
 
       if opts.respond_to? :deep_symbolize_keys!
         opts.deep_symbolize_keys!
@@ -368,6 +391,8 @@ module Sidekiq
       end
 
       opts = opts.merge(opts.delete(environment.to_sym) || {})
+      opts.delete(:strict)
+
       parse_queues(opts, opts.delete(:queues) || [])
 
       opts
@@ -379,8 +404,9 @@ module Sidekiq
 
     def parse_queue(opts, queue, weight = nil)
       opts[:queues] ||= []
+      opts[:strict] = true if opts[:strict].nil?
       raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
-      [weight.to_i, 1].max.times { opts[:queues] << queue }
+      [weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
       opts[:strict] = false if weight.to_i > 0
     end
 

data/lib/sidekiq/client.rb

@@ -2,9 +2,12 @@
 
 require "securerandom"
 require "sidekiq/middleware/chain"
+require "sidekiq/job_util"
 
 module Sidekiq
   class Client
+    include Sidekiq::JobUtil
+
     ##
     # Define client-side middleware:
     #
@@ -19,7 +22,7 @@ module Sidekiq
     #
     def middleware(&block)
       @chain ||= Sidekiq.client_middleware
-      if block_given?
+      if block
         @chain = @chain.dup
         yield @chain
       end
@@ -90,16 +93,17 @@ module Sidekiq
     # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
     # than the number given if the middleware stopped processing for one or more jobs.
     def push_bulk(items)
-      arg = items["args"].first
-      return [] unless arg # no jobs to push
-      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless arg.is_a?(Array)
+      args = items["args"]
+      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
+      return [] if args.empty? # no jobs to push
 
       at = items.delete("at")
-      raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
+      raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
+      raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
       normed = normalize_item(items)
-      payloads = items["args"].map.with_index { |args, index|
-        copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
+      payloads = args.map.with_index { |job_args, index|
+        copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
         copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
 
         result = process_single(items["class"], copy)
@@ -185,7 +189,7 @@ module Sidekiq
 
     def raw_push(payloads)
       @redis_pool.with do |conn|
-        conn.multi do
+        conn.pipelined do
           atomic_push(conn, payloads)
         end
       end
@@ -217,42 +221,5 @@ module Sidekiq
         item
       end
     end
-
-    def normalize_item(item)
-      # 6.0.0 push_bulk bug, #4321
-      # TODO Remove after a while...
-      item.delete("at") if item.key?("at") && item["at"].nil?
-
-      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
-      raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
-      raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
-      raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
-      raise(ArgumentError, "Job tags must be an Array") if item["tags"] && !item["tags"].is_a?(Array)
-      # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
-
-      # merge in the default sidekiq_options for the item's class and/or wrapped element
-      # this allows ActiveJobs to control sidekiq_options too.
-      defaults = normalized_hash(item["class"])
-      defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
-      item = defaults.merge(item)
-
-      raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
-
-      item["class"] = item["class"].to_s
-      item["queue"] = item["queue"].to_s
-      item["jid"] ||= SecureRandom.hex(12)
-      item["created_at"] ||= Time.now.to_f
-
-      item
-    end
-
-    def normalized_hash(item_class)
-      if item_class.is_a?(Class)
-        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
-        item_class.get_sidekiq_options
-      else
-        Sidekiq.default_worker_options
-      end
-    end
   end
 end

data/lib/sidekiq/delay.rb

@@ -3,6 +3,8 @@
 module Sidekiq
   module Extensions
     def self.enable_delay!
+      Sidekiq.logger.error "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0. #{caller(1..1).first}"
+
       if defined?(::ActiveSupport)
         require "sidekiq/extensions/active_record"
         require "sidekiq/extensions/action_mailer"

data/lib/sidekiq/extensions/action_mailer.rb

@@ -5,9 +5,10 @@ require "sidekiq/extensions/generic_proxy"
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to ActionMailer to offload arbitrary email
-    # delivery to Sidekiq.  Example:
+    # Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email
+    # delivery to Sidekiq.
     #
+    # @example
     #    UserMailer.delay.send_welcome_email(new_user)
     #    UserMailer.delay_for(5.days).send_welcome_email(new_user)
     #    UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
@@ -15,8 +16,8 @@ module Sidekiq
       include Sidekiq::Worker
 
       def perform(yml)
-        (target, method_name, args) = YAML.load(yml)
-        msg = target.public_send(method_name, *args)
+        (target, method_name, args, kwargs) = YAML.load(yml)
+        msg = kwargs.empty? ? target.public_send(method_name, *args) : target.public_send(method_name, *args, **kwargs)
         # The email method can return nil, which causes ActionMailer to return
         # an undeliverable empty message.
         if msg

data/lib/sidekiq/extensions/active_record.rb

@@ -5,10 +5,11 @@ require "sidekiq/extensions/generic_proxy"
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to ActiveRecord to offload instance method
-    # execution to Sidekiq.  Examples:
+    # Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method
+    # execution to Sidekiq.
     #
-    # User.recent_signups.each { |user| user.delay.mark_as_awesome }
+    # @example
+    #   User.recent_signups.each { |user| user.delay.mark_as_awesome }
     #
     # Please note, this is not recommended as this will serialize the entire
     # object to Redis.  Your Sidekiq jobs should pass IDs, not entire instances.
@@ -17,8 +18,8 @@ module Sidekiq
       include Sidekiq::Worker
 
       def perform(yml)
-        (target, method_name, args) = YAML.load(yml)
-        target.__send__(method_name, *args)
+        (target, method_name, args, kwargs) = YAML.load(yml)
+        kwargs.empty? ? target.__send__(method_name, *args) : target.__send__(method_name, *args, **kwargs)
       end
     end
 

data/lib/sidekiq/extensions/class_methods.rb

@@ -5,18 +5,19 @@ require "sidekiq/extensions/generic_proxy"
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to all Classes to offload class method
-    # execution to Sidekiq.  Examples:
+    # Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method
+    # execution to Sidekiq.
     #
-    # User.delay.delete_inactive
-    # Wikipedia.delay.download_changes_for(Date.today)
+    # @example
+    #   User.delay.delete_inactive
+    #   Wikipedia.delay.download_changes_for(Date.today)
     #
     class DelayedClass
       include Sidekiq::Worker
 
       def perform(yml)
-        (target, method_name, args) = YAML.load(yml)
-        target.__send__(method_name, *args)
+        (target, method_name, args, kwargs) = YAML.load(yml)
+        kwargs.empty? ? target.__send__(method_name, *args) : target.__send__(method_name, *args, **kwargs)
       end
     end
 

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -13,18 +13,20 @@ module Sidekiq
         @opts = options
       end
 
-      def method_missing(name, *args)
+      def method_missing(name, *args, **kwargs)
         # Sidekiq has a limitation in that its message must be JSON.
         # JSON can't round trip real Ruby objects so we use YAML to
         # serialize the objects to a String.  The YAML will be converted
         # to JSON and then deserialized on the other side back into a
         # Ruby object.
-        obj = [@target, name, args]
+        obj = [@target, name, args, kwargs]
         marshalled = ::YAML.dump(obj)
         if marshalled.size > SIZE_LIMIT
           ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
         end
-        @performable.client_push({"class" => @performable, "args" => [marshalled]}.merge(@opts))
+        @performable.client_push({"class" => @performable,
+                                  "args" => [marshalled],
+                                  "display_class" => "#{@target}.#{name}"}.merge(@opts))
       end
     end
   end

data/lib/sidekiq/fetch.rb

@@ -25,8 +25,10 @@ module Sidekiq
     }
 
     def initialize(options)
-      @strictly_ordered_queues = !!options[:strict]
-      @queues = options[:queues].map { |q| "queue:#{q}" }
+      raise ArgumentError, "missing queue list" unless options[:queues]
+      @options = options
+      @strictly_ordered_queues = !!@options[:strict]
+      @queues = @options[:queues].map { |q| "queue:#{q}" }
       if @strictly_ordered_queues
         @queues.uniq!
         @queues << TIMEOUT
@@ -34,28 +36,19 @@ module Sidekiq
     end
 
     def retrieve_work
-      work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
-      UnitOfWork.new(*work) if work
-    end
-
-    # Creating the Redis#brpop command takes into account any
-    # configured queue weights. By default Redis#brpop returns
-    # data from the first queue that has pending elements. We
-    # recreate the queue command each time we invoke Redis#brpop
-    # to honor weights and avoid queue starvation.
-    def queues_cmd
-      if @strictly_ordered_queues
-        @queues
-      else
-        queues = @queues.shuffle!.uniq
-        queues << TIMEOUT
-        queues
+      qs = queues_cmd
+      # 4825 Sidekiq Pro with all queues paused will return an
+      # empty set of queues with a trailing TIMEOUT value.
+      if qs.size <= 1
+        sleep(TIMEOUT)
+        return nil
       end
+
+      work = Sidekiq.redis { |conn| conn.brpop(*qs) }
+      UnitOfWork.new(*work) if work
     end
 
-    # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
-    # an instance method will make it async to the Fetcher actor
-    def self.bulk_requeue(inprogress, options)
+    def bulk_requeue(inprogress, options)
       return if inprogress.empty?
 
       Sidekiq.logger.debug { "Re-queueing terminated jobs" }
@@ -76,5 +69,21 @@ module Sidekiq
     rescue => ex
       Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
     end
+
+    # Creating the Redis#brpop command takes into account any
+    # configured queue weights. By default Redis#brpop returns
+    # data from the first queue that has pending elements. We
+    # recreate the queue command each time we invoke Redis#brpop
+    # to honor weights and avoid queue starvation.
+    def queues_cmd
+      if @strictly_ordered_queues
+        @queues
+      else
+        permute = @queues.shuffle
+        permute.uniq!
+        permute << TIMEOUT
+        permute
+      end
+    end
   end
 end

data/lib/sidekiq/job.rb

@@ -0,0 +1,13 @@
+require "sidekiq/worker"
+
+module Sidekiq
+  # Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
+  # Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
+  #
+  # The term "worker" is too generic and overly confusing, used in several
+  # different contexts meaning different things. Many people call a Sidekiq
+  # process a "worker". Some people call the thread that executes jobs a
+  # "worker". This change brings Sidekiq closer to ActiveJob where your job
+  # classes extend ApplicationJob.
+  Job = Worker
+end

data/lib/sidekiq/job_logger.rb

@@ -38,7 +38,7 @@ module Sidekiq
       # If we're using a wrapper class, like ActiveJob, use the "wrapped"
       # attribute to expose the underlying thing.
       h = {
-        class: job_hash["wrapped"] || job_hash["class"],
+        class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
         jid: job_hash["jid"]
       }
       h[:bid] = job_hash["bid"] if job_hash["bid"]

data/lib/sidekiq/job_retry.rb

@@ -34,9 +34,10 @@ module Sidekiq
   # The job will be retried this number of times before giving up. (If simply
   # 'true', Sidekiq retries 25 times)
   #
-  # We'll add a bit more data to the job to support retries:
+  # Relevant options for job retries:
   #
-  #  * 'queue' - the queue to use
+  #  * 'queue' - the queue for the initial job
+  #  * 'retry_queue' - if job retries should be pushed to a different (e.g. lower priority) queue
   #  * 'retry_count' - number of times we've retried so far.
   #  * 'error_message' - the message from the exception
   #  * 'error_class' - the exception class
@@ -52,15 +53,17 @@ module Sidekiq
   #
   #   Sidekiq.options[:max_retries] = 7
   #
-  # or limit the number of retries for a particular worker with:
+  # or limit the number of retries for a particular worker and send retries to
+  # a low priority queue with:
   #
   #    class MyWorker
   #      include Sidekiq::Worker
-  #      sidekiq_options :retry => 10
+  #      sidekiq_options retry: 10, retry_queue: 'low'
   #    end
   #
   class JobRetry
     class Handled < ::RuntimeError; end
+
     class Skip < Handled; end
 
     include Sidekiq::Util
@@ -213,16 +216,12 @@ module Sidekiq
     end
 
     def delay_for(worker, count, exception)
+      jitter = rand(10) * (count + 1)
       if worker&.sidekiq_retry_in_block
         custom_retry_in = retry_in(worker, count, exception).to_i
-        return custom_retry_in if custom_retry_in > 0
+        return custom_retry_in + jitter if custom_retry_in > 0
       end
-      seconds_to_delay(count)
-    end
-
-    # delayed_job uses the same basic formula
-    def seconds_to_delay(count)
-      (count**4) + 15 + (rand(30) * (count + 1))
+      (count**4) + 15 + jitter
     end
 
     def retry_in(worker, count, exception)

data/lib/sidekiq/job_util.rb

@@ -0,0 +1,65 @@
+require "securerandom"
+require "time"
+
+module Sidekiq
+  module JobUtil
+    # These functions encapsulate various job utilities.
+    # They must be simple and free from side effects.
+
+    def validate(item)
+      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
+      raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
+      raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
+      raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
+      raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
+
+      if Sidekiq.options[:on_complex_arguments] == :raise
+        msg = <<~EOM
+          Job arguments to #{item["class"]} must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices.
+          To disable this error, remove `Sidekiq.strict_args!` from your initializer.
+        EOM
+        raise(ArgumentError, msg) unless json_safe?(item)
+      elsif Sidekiq.options[:on_complex_arguments] == :warn
+        Sidekiq.logger.warn <<~EOM unless json_safe?(item)
+          Job arguments to #{item["class"]} do not serialize to JSON safely. This will raise an error in
+          Sidekiq 7.0. See https://github.com/mperham/sidekiq/wiki/Best-Practices or raise an error today
+          by calling `Sidekiq.strict_args!` during Sidekiq initialization.
+        EOM
+      end
+    end
+
+    def normalize_item(item)
+      validate(item)
+
+      # merge in the default sidekiq_options for the item's class and/or wrapped element
+      # this allows ActiveJobs to control sidekiq_options too.
+      defaults = normalized_hash(item["class"])
+      defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?(:get_sidekiq_options)
+      item = defaults.merge(item)
+
+      raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
+
+      item["class"] = item["class"].to_s
+      item["queue"] = item["queue"].to_s
+      item["jid"] ||= SecureRandom.hex(12)
+      item["created_at"] ||= Time.now.to_f
+
+      item
+    end
+
+    def normalized_hash(item_class)
+      if item_class.is_a?(Class)
+        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?(:get_sidekiq_options)
+        item_class.get_sidekiq_options
+      else
+        Sidekiq.default_worker_options
+      end
+    end
+
+    private
+
+    def json_safe?(item)
+      JSON.parse(JSON.dump(item["args"])) == item["args"]
+    end
+  end
+end