sidekiq 6.0.0 → 6.4.0

data/lib/sidekiq/cli.rb

@@ -33,17 +33,29 @@ module Sidekiq
     # Code within this method is not tested because it alters
     # global process state irreversibly.  PRs which improve the
     # test coverage of Sidekiq::CLI are welcomed.
-    def run
-      boot_system
+    def run(boot_app: true)
+      boot_application if boot_app
+
       if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
         print_banner
       end
+      logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
 
       self_read, self_write = IO.pipe
       sigs = %w[INT TERM TTIN TSTP]
+      # USR1 and USR2 don't work on the JVM
+      sigs << "USR2" if Sidekiq.pro? && !jruby?
       sigs.each do |sig|
-        trap sig do
-          self_write.write("#{sig}\n")
+        old_handler = Signal.trap(sig) do
+          if old_handler.respond_to?(:call)
+            begin
+              old_handler.call
+            rescue Exception => exc
+              # signal handlers can't use Logger so puts only
+              puts ["Error in #{sig} handler", exc].inspect
+            end
+          end
+          self_write.puts(sig)
         end
       rescue ArgumentError
         puts "Signal #{sig} not supported"
@@ -51,12 +63,25 @@ module Sidekiq
 
       logger.info "Running in #{RUBY_DESCRIPTION}"
       logger.info Sidekiq::LICENSE
-      logger.info "Upgrade to Sidekiq Pro for more features and support: http://sidekiq.org" unless defined?(::Sidekiq::Pro)
+      logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
 
       # touch the connection pool so it is created before we
       # fire startup and start multithreading.
-      ver = Sidekiq.redis_info["redis_version"]
-      raise "You are using Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
+      info = Sidekiq.redis_info
+      ver = info["redis_version"]
+      raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
+
+      maxmemory_policy = info["maxmemory_policy"]
+      if maxmemory_policy != "noeviction"
+        logger.warn <<~EOM
+
+
+          WARNING: Your Redis instance will evict Sidekiq data under heavy load.
+          The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
+          See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
+
+        EOM
+      end
 
       # Since the user can pass us a connection pool explicitly in the initializer, we
       # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
@@ -160,17 +185,14 @@ module Sidekiq
             Sidekiq.logger.warn "<no backtrace available>"
           end
         end
-      },
+      }
     }
+    UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
+    SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
 
     def handle_signal(sig)
       Sidekiq.logger.debug "Got #{sig} signal"
-      handy = SIGNAL_HANDLERS[sig]
-      if handy
-        handy.call(self)
-      else
-        Sidekiq.logger.info { "No signal handler for #{sig}" }
-      end
+      SIGNAL_HANDLERS[sig].call(self)
     end
 
     private
@@ -182,7 +204,11 @@ module Sidekiq
     end
 
     def set_environment(cli_env)
-      @environment = cli_env || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      # See #984 for discussion.
+      # APP_ENV is now the preferred ENV term since it is not tech-specific.
+      # Both Sinatra 2.0+ and Sidekiq support this term.
+      # RAILS_ENV and RACK_ENV are there for legacy support.
+      @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
     end
 
     def symbolize_keys_deep!(hash)
@@ -204,7 +230,7 @@ module Sidekiq
 
       # check config file presence
       if opts[:config_file]
-        if opts[:config_file] && !File.exist?(opts[:config_file])
+        unless File.exist?(opts[:config_file])
           raise ArgumentError, "No such file #{opts[:config_file]}"
         end
       else
@@ -224,8 +250,7 @@ module Sidekiq
       opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
 
       # set defaults
-      opts[:queues] = Array(opts[:queues]) << "default" if opts[:queues].nil? || opts[:queues].empty?
-      opts[:strict] = true if opts[:strict].nil?
+      opts[:queues] = ["default"] if opts[:queues].nil?
       opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
 
       # merge with defaults
@@ -236,7 +261,7 @@ module Sidekiq
       Sidekiq.options
     end
 
-    def boot_system
+    def boot_application
       ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
 
       if File.directory?(options[:require])
@@ -283,8 +308,13 @@ module Sidekiq
 
     def parse_options(argv)
       opts = {}
+      @parser = option_parser(opts)
+      @parser.parse!(argv)
+      opts
+    end
 
-      @parser = OptionParser.new { |o|
+    def option_parser(opts)
+      parser = OptionParser.new { |o|
         o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
           opts[:concurrency] = Integer(arg)
         end
@@ -336,15 +366,13 @@ module Sidekiq
         end
       }
 
-      @parser.banner = "sidekiq [options]"
-      @parser.on_tail "-h", "--help", "Show help" do
-        logger.info @parser
+      parser.banner = "sidekiq [options]"
+      parser.on_tail "-h", "--help", "Show help" do
+        logger.info parser
         die 1
       end
 
-      @parser.parse!(argv)
-
-      opts
+      parser
     end
 
     def initialize_logger
@@ -352,7 +380,9 @@ module Sidekiq
     end
 
     def parse_config(path)
-      opts = YAML.load(ERB.new(File.read(path)).result) || {}
+      erb = ERB.new(File.read(path))
+      erb.filename = File.expand_path(path)
+      opts = YAML.load(erb.result) || {}
 
       if opts.respond_to? :deep_symbolize_keys!
         opts.deep_symbolize_keys!
@@ -361,6 +391,8 @@ module Sidekiq
       end
 
       opts = opts.merge(opts.delete(environment.to_sym) || {})
+      opts.delete(:strict)
+
       parse_queues(opts, opts.delete(:queues) || [])
 
       opts
@@ -372,9 +404,16 @@ module Sidekiq
 
     def parse_queue(opts, queue, weight = nil)
       opts[:queues] ||= []
+      opts[:strict] = true if opts[:strict].nil?
       raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
-      [weight.to_i, 1].max.times { opts[:queues] << queue }
+      [weight.to_i, 1].max.times { opts[:queues] << queue.to_s }
       opts[:strict] = false if weight.to_i > 0
     end
+
+    def rails_app?
+      defined?(::Rails) && ::Rails.respond_to?(:application)
+    end
   end
 end
+
+require "sidekiq/systemd"

data/lib/sidekiq/client.rb

@@ -2,9 +2,12 @@
 
 require "securerandom"
 require "sidekiq/middleware/chain"
+require "sidekiq/job_util"
 
 module Sidekiq
   class Client
+    include Sidekiq::JobUtil
+
     ##
     # Define client-side middleware:
     #
@@ -19,7 +22,7 @@ module Sidekiq
     #
     def middleware(&block)
       @chain ||= Sidekiq.client_middleware
-      if block_given?
+      if block
         @chain = @chain.dup
         yield @chain
       end
@@ -90,13 +93,19 @@ module Sidekiq
     # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
     # than the number given if the middleware stopped processing for one or more jobs.
     def push_bulk(items)
-      arg = items["args"].first
-      return [] unless arg # no jobs to push
-      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless arg.is_a?(Array)
+      args = items["args"]
+      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
+      return [] if args.empty? # no jobs to push
+
+      at = items.delete("at")
+      raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
+      raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
       normed = normalize_item(items)
-      payloads = items["args"].map { |args|
-        copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
+      payloads = args.map.with_index { |job_args, index|
+        copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
+        copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
+
         result = process_single(items["class"], copy)
         result || nil
       }.compact
@@ -180,7 +189,7 @@ module Sidekiq
 
     def raw_push(payloads)
       @redis_pool.with do |conn|
-        conn.multi do
+        conn.pipelined do
           atomic_push(conn, payloads)
         end
       end
@@ -188,7 +197,7 @@ module Sidekiq
     end
 
     def atomic_push(conn, payloads)
-      if payloads.first["at"]
+      if payloads.first.key?("at")
         conn.zadd("schedule", payloads.map { |hash|
           at = hash.delete("at").to_s
           [at, Sidekiq.dump_json(hash)]
@@ -212,31 +221,5 @@ module Sidekiq
         item
       end
     end
-
-    def normalize_item(item)
-      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
-      raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
-      raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
-      raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
-      # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
-
-      normalized_hash(item["class"])
-        .each { |key, value| item[key] = value if item[key].nil? }
-
-      item["class"] = item["class"].to_s
-      item["queue"] = item["queue"].to_s
-      item["jid"] ||= SecureRandom.hex(12)
-      item["created_at"] ||= Time.now.to_f
-      item
-    end
-
-    def normalized_hash(item_class)
-      if item_class.is_a?(Class)
-        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
-        item_class.get_sidekiq_options
-      else
-        Sidekiq.default_worker_options
-      end
-    end
   end
 end

data/lib/sidekiq/delay.rb

@@ -3,6 +3,8 @@
 module Sidekiq
   module Extensions
     def self.enable_delay!
+      Sidekiq.logger.error "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0. #{caller(1..1).first}"
+
       if defined?(::ActiveSupport)
         require "sidekiq/extensions/active_record"
         require "sidekiq/extensions/action_mailer"

data/lib/sidekiq/extensions/action_mailer.rb

@@ -5,9 +5,10 @@ require "sidekiq/extensions/generic_proxy"
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to ActionMailer to offload arbitrary email
-    # delivery to Sidekiq.  Example:
+    # Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email
+    # delivery to Sidekiq.
     #
+    # @example
     #    UserMailer.delay.send_welcome_email(new_user)
     #    UserMailer.delay_for(5.days).send_welcome_email(new_user)
     #    UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
@@ -15,8 +16,8 @@ module Sidekiq
       include Sidekiq::Worker
 
       def perform(yml)
-        (target, method_name, args) = YAML.load(yml)
-        msg = target.public_send(method_name, *args)
+        (target, method_name, args, kwargs) = YAML.load(yml)
+        msg = kwargs.empty? ? target.public_send(method_name, *args) : target.public_send(method_name, *args, **kwargs)
         # The email method can return nil, which causes ActionMailer to return
         # an undeliverable empty message.
         if msg

data/lib/sidekiq/extensions/active_record.rb

@@ -5,10 +5,11 @@ require "sidekiq/extensions/generic_proxy"
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to ActiveRecord to offload instance method
-    # execution to Sidekiq.  Examples:
+    # Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method
+    # execution to Sidekiq.
     #
-    # User.recent_signups.each { |user| user.delay.mark_as_awesome }
+    # @example
+    #   User.recent_signups.each { |user| user.delay.mark_as_awesome }
     #
     # Please note, this is not recommended as this will serialize the entire
     # object to Redis.  Your Sidekiq jobs should pass IDs, not entire instances.
@@ -17,8 +18,8 @@ module Sidekiq
       include Sidekiq::Worker
 
       def perform(yml)
-        (target, method_name, args) = YAML.load(yml)
-        target.__send__(method_name, *args)
+        (target, method_name, args, kwargs) = YAML.load(yml)
+        kwargs.empty? ? target.__send__(method_name, *args) : target.__send__(method_name, *args, **kwargs)
       end
     end
 

data/lib/sidekiq/extensions/class_methods.rb

@@ -5,18 +5,19 @@ require "sidekiq/extensions/generic_proxy"
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to all Classes to offload class method
-    # execution to Sidekiq.  Examples:
+    # Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method
+    # execution to Sidekiq.
     #
-    # User.delay.delete_inactive
-    # Wikipedia.delay.download_changes_for(Date.today)
+    # @example
+    #   User.delay.delete_inactive
+    #   Wikipedia.delay.download_changes_for(Date.today)
     #
     class DelayedClass
       include Sidekiq::Worker
 
       def perform(yml)
-        (target, method_name, args) = YAML.load(yml)
-        target.__send__(method_name, *args)
+        (target, method_name, args, kwargs) = YAML.load(yml)
+        kwargs.empty? ? target.__send__(method_name, *args) : target.__send__(method_name, *args, **kwargs)
       end
     end
 

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -13,18 +13,20 @@ module Sidekiq
         @opts = options
       end
 
-      def method_missing(name, *args)
+      def method_missing(name, *args, **kwargs)
         # Sidekiq has a limitation in that its message must be JSON.
         # JSON can't round trip real Ruby objects so we use YAML to
         # serialize the objects to a String.  The YAML will be converted
         # to JSON and then deserialized on the other side back into a
         # Ruby object.
-        obj = [@target, name, args]
+        obj = [@target, name, args, kwargs]
         marshalled = ::YAML.dump(obj)
         if marshalled.size > SIZE_LIMIT
           ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
         end
-        @performable.client_push({"class" => @performable, "args" => [marshalled]}.merge(@opts))
+        @performable.client_push({"class" => @performable,
+                                  "args" => [marshalled],
+                                  "display_class" => "#{@target}.#{name}"}.merge(@opts))
       end
     end
   end

data/lib/sidekiq/fetch.rb

@@ -14,61 +14,54 @@ module Sidekiq
       end
 
       def queue_name
-        queue.sub(/.*queue:/, "")
+        queue.delete_prefix("queue:")
       end
 
       def requeue
         Sidekiq.redis do |conn|
-          conn.rpush("queue:#{queue_name}", job)
+          conn.rpush(queue, job)
         end
       end
     }
 
     def initialize(options)
-      @strictly_ordered_queues = !!options[:strict]
-      @queues = options[:queues].map { |q| "queue:#{q}" }
+      raise ArgumentError, "missing queue list" unless options[:queues]
+      @options = options
+      @strictly_ordered_queues = !!@options[:strict]
+      @queues = @options[:queues].map { |q| "queue:#{q}" }
       if @strictly_ordered_queues
-        @queues = @queues.uniq
+        @queues.uniq!
         @queues << TIMEOUT
       end
     end
 
     def retrieve_work
-      work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
-      UnitOfWork.new(*work) if work
-    end
-
-    # Creating the Redis#brpop command takes into account any
-    # configured queue weights. By default Redis#brpop returns
-    # data from the first queue that has pending elements. We
-    # recreate the queue command each time we invoke Redis#brpop
-    # to honor weights and avoid queue starvation.
-    def queues_cmd
-      if @strictly_ordered_queues
-        @queues
-      else
-        queues = @queues.shuffle.uniq
-        queues << TIMEOUT
-        queues
+      qs = queues_cmd
+      # 4825 Sidekiq Pro with all queues paused will return an
+      # empty set of queues with a trailing TIMEOUT value.
+      if qs.size <= 1
+        sleep(TIMEOUT)
+        return nil
       end
+
+      work = Sidekiq.redis { |conn| conn.brpop(*qs) }
+      UnitOfWork.new(*work) if work
     end
 
-    # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
-    # an instance method will make it async to the Fetcher actor
-    def self.bulk_requeue(inprogress, options)
+    def bulk_requeue(inprogress, options)
       return if inprogress.empty?
 
       Sidekiq.logger.debug { "Re-queueing terminated jobs" }
       jobs_to_requeue = {}
       inprogress.each do |unit_of_work|
-        jobs_to_requeue[unit_of_work.queue_name] ||= []
-        jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.job
+        jobs_to_requeue[unit_of_work.queue] ||= []
+        jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
       end
 
       Sidekiq.redis do |conn|
         conn.pipelined do
           jobs_to_requeue.each do |queue, jobs|
-            conn.rpush("queue:#{queue}", jobs)
+            conn.rpush(queue, jobs)
           end
         end
       end
@@ -76,5 +69,21 @@ module Sidekiq
     rescue => ex
       Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
     end
+
+    # Creating the Redis#brpop command takes into account any
+    # configured queue weights. By default Redis#brpop returns
+    # data from the first queue that has pending elements. We
+    # recreate the queue command each time we invoke Redis#brpop
+    # to honor weights and avoid queue starvation.
+    def queues_cmd
+      if @strictly_ordered_queues
+        @queues
+      else
+        permute = @queues.shuffle
+        permute.uniq!
+        permute << TIMEOUT
+        permute
+      end
+    end
   end
 end

data/lib/sidekiq/job.rb

@@ -0,0 +1,13 @@
+require "sidekiq/worker"
+
+module Sidekiq
+  # Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
+  # Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
+  #
+  # The term "worker" is too generic and overly confusing, used in several
+  # different contexts meaning different things. Many people call a Sidekiq
+  # process a "worker". Some people call the thread that executes jobs a
+  # "worker". This change brings Sidekiq closer to ActiveJob where your job
+  # classes extend ApplicationJob.
+  Job = Worker
+end

data/lib/sidekiq/job_logger.rb

@@ -23,23 +23,31 @@ module Sidekiq
       raise
     end
 
-    def with_job_hash_context(job_hash, &block)
-      @logger.with_context(job_hash_context(job_hash), &block)
+    def prepare(job_hash, &block)
+      level = job_hash["log_level"]
+      if level
+        @logger.log_at(level) do
+          Sidekiq::Context.with(job_hash_context(job_hash), &block)
+        end
+      else
+        Sidekiq::Context.with(job_hash_context(job_hash), &block)
+      end
     end
 
     def job_hash_context(job_hash)
       # If we're using a wrapper class, like ActiveJob, use the "wrapped"
       # attribute to expose the underlying thing.
       h = {
-        class: job_hash["wrapped"] || job_hash["class"],
-        jid: job_hash["jid"],
+        class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
+        jid: job_hash["jid"]
       }
       h[:bid] = job_hash["bid"] if job_hash["bid"]
+      h[:tags] = job_hash["tags"] if job_hash["tags"]
       h
     end
 
     def with_elapsed_time_context(start, &block)
-      @logger.with_context(elapsed_time_context(start), &block)
+      Sidekiq::Context.with(elapsed_time_context(start), &block)
     end
 
     def elapsed_time_context(start)