sidekiq 6.3.1 → 6.4.2

data/lib/sidekiq/cli.rb

@@ -20,7 +20,7 @@ module Sidekiq
     attr_accessor :launcher
     attr_accessor :environment
 
-    def parse(args = ARGV)
+    def parse(args = ARGV.dup)
       setup_options(args)
       initialize_logger
       validate!
@@ -115,8 +115,8 @@ module Sidekiq
       begin
         launcher.run
 
-        while (readable_io = IO.select([self_read]))
-          signal = readable_io.first[0].gets.strip
+        while self_read.wait_readable
+          signal = self_read.gets.strip
           handle_signal(signal)
         end
       rescue Interrupt
@@ -295,7 +295,7 @@ module Sidekiq
           (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
         logger.info "=================================================================="
         logger.info "  Please point Sidekiq to a Rails application or a Ruby file  "
-        logger.info "  to load your worker classes with -r [DIR|FILE]."
+        logger.info "  to load your job classes with -r [DIR|FILE]."
         logger.info "=================================================================="
         logger.info @parser
         die(1)
@@ -336,7 +336,7 @@ module Sidekiq
           parse_queue opts, queue, weight
         end
 
-        o.on "-r", "--require [PATH|DIR]", "Location of Rails application with workers or file to require" do |arg|
+        o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
           opts[:require] = arg
         end
 
@@ -380,7 +380,9 @@ module Sidekiq
     end
 
     def parse_config(path)
-      opts = YAML.load(ERB.new(File.read(path)).result) || {}
+      erb = ERB.new(File.read(path))
+      erb.filename = File.expand_path(path)
+      opts = load_yaml(erb.result) || {}
 
       if opts.respond_to? :deep_symbolize_keys!
         opts.deep_symbolize_keys!
@@ -396,6 +398,14 @@ module Sidekiq
       opts
     end
 
+    def load_yaml(src)
+      if Psych::VERSION > "4.0"
+        YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
+      else
+        YAML.load(src)
+      end
+    end
+
     def parse_queues(opts, queues_and_weights)
       queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
     end

data/lib/sidekiq/client.rb

@@ -2,9 +2,12 @@
 
 require "securerandom"
 require "sidekiq/middleware/chain"
+require "sidekiq/job_util"
 
 module Sidekiq
   class Client
+    include Sidekiq::JobUtil
+
     ##
     # Define client-side middleware:
     #
@@ -12,7 +15,7 @@ module Sidekiq
     #   client.middleware do |chain|
     #     chain.use MyClientMiddleware
     #   end
-    #   client.push('class' => 'SomeWorker', 'args' => [1,2,3])
+    #   client.push('class' => 'SomeJob', 'args' => [1,2,3])
     #
     # All client instances default to the globally-defined
     # Sidekiq.client_middleware but you can change as necessary.
@@ -46,16 +49,16 @@ module Sidekiq
     # The main method used to push a job to Redis.  Accepts a number of options:
     #
     #   queue - the named queue to use, default 'default'
-    #   class - the worker class to call, required
+    #   class - the job class to call, required
     #   args - an array of simple arguments to the perform method, must be JSON-serializable
     #   at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
     #   retry - whether to retry this job if it fails, default true or an integer number of retries
     #   backtrace - whether to save any error backtrace, default false
     #
     # If class is set to the class name, the jobs' options will be based on Sidekiq's default
-    # worker options. Otherwise, they will be based on the job class's options.
+    # job options. Otherwise, they will be based on the job class's options.
     #
-    # Any options valid for a worker class's sidekiq_options are also available here.
+    # Any options valid for a job class's sidekiq_options are also available here.
     #
     # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
     # symbols in 'args' will be converted to strings.  Note that +backtrace: true+ can take quite a bit of
@@ -64,13 +67,15 @@ module Sidekiq
     # Returns a unique Job ID.  If middleware stops the job, nil will be returned instead.
     #
     # Example:
-    #   push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
+    #   push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
     #
     def push(item)
       normed = normalize_item(item)
-      payload = process_single(item["class"], normed)
-
+      payload = middleware.invoke(normed["class"], normed, normed["queue"], @redis_pool) do
+        normed
+      end
       if payload
+        verify_json(payload)
         raw_push([payload])
         payload["jid"]
       end
@@ -98,12 +103,17 @@ module Sidekiq
       raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
       raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
+      jid = items.delete("jid")
+      raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
+
       normed = normalize_item(items)
       payloads = args.map.with_index { |job_args, index|
-        copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
+        copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
         copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
-
-        result = process_single(items["class"], copy)
+        result = middleware.invoke(copy["class"], copy, copy["queue"], @redis_pool) do
+          verify_json(copy)
+          copy
+        end
         result || nil
       }.compact
 
@@ -116,8 +126,8 @@ module Sidekiq
     #
     #   pool = ConnectionPool.new { Redis.new }
     #   Sidekiq::Client.via(pool) do
-    #     SomeWorker.perform_async(1,2,3)
-    #     SomeOtherWorker.perform_async(1,2,3)
+    #     SomeJob.perform_async(1,2,3)
+    #     SomeOtherJob.perform_async(1,2,3)
     #   end
     #
     # Generally this is only needed for very large Sidekiq installs processing
@@ -142,10 +152,10 @@ module Sidekiq
       end
 
       # Resque compatibility helpers.  Note all helpers
-      # should go through Worker#client_push.
+      # should go through Sidekiq::Job#client_push.
       #
       # Example usage:
-      #   Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
       #
       # Messages are enqueued to the 'default' queue.
       #
@@ -154,14 +164,14 @@ module Sidekiq
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to(queue, klass, *args)
         klass.client_push("queue" => queue, "class" => klass, "args" => args)
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to_in(queue, interval, klass, *args)
         int = interval.to_f
@@ -175,7 +185,7 @@ module Sidekiq
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_in(interval, klass, *args)
         klass.perform_in(interval, *args)
@@ -186,8 +196,23 @@ module Sidekiq
 
     def raw_push(payloads)
       @redis_pool.with do |conn|
-        conn.pipelined do
-          atomic_push(conn, payloads)
+        retryable = true
+        begin
+          conn.pipelined do |pipeline|
+            atomic_push(pipeline, payloads)
+          end
+        rescue Redis::BaseError => ex
+          # 2550 Failover can cause the server to become a replica, need
+          # to disconnect and reopen the socket to get back to the primary.
+          # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
+          # 4985 Use the same logic when a blocking command is force-unblocked
+          # The retry logic is copied from sidekiq.rb
+          if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
+            conn.disconnect!
+            retryable = false
+            retry
+          end
+          raise
         end
       end
       true
@@ -210,50 +235,5 @@ module Sidekiq
         conn.lpush("queue:#{queue}", to_push)
       end
     end
-
-    def process_single(worker_class, item)
-      queue = item["queue"]
-
-      middleware.invoke(worker_class, item, queue, @redis_pool) do
-        item
-      end
-    end
-
-    def validate(item)
-      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
-      raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
-      raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
-      raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
-      raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
-    end
-
-    def normalize_item(item)
-      validate(item)
-      # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
-
-      # merge in the default sidekiq_options for the item's class and/or wrapped element
-      # this allows ActiveJobs to control sidekiq_options too.
-      defaults = normalized_hash(item["class"])
-      defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
-      item = defaults.merge(item)
-
-      raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
-
-      item["class"] = item["class"].to_s
-      item["queue"] = item["queue"].to_s
-      item["jid"] ||= SecureRandom.hex(12)
-      item["created_at"] ||= Time.now.to_f
-
-      item
-    end
-
-    def normalized_hash(item_class)
-      if item_class.is_a?(Class)
-        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
-        item_class.get_sidekiq_options
-      else
-        Sidekiq.default_worker_options
-      end
-    end
   end
 end

data/lib/sidekiq/delay.rb

@@ -3,6 +3,8 @@
 module Sidekiq
   module Extensions
     def self.enable_delay!
+      warn "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0", uplevel: 1
+
       if defined?(::ActiveSupport)
         require "sidekiq/extensions/active_record"
         require "sidekiq/extensions/action_mailer"

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -10,7 +10,7 @@ module Sidekiq
       def initialize(performable, target, options = {})
         @performable = performable
         @target = target
-        @opts = options
+        @opts = options.transform_keys(&:to_s)
       end
 
       def method_missing(name, *args)

data/lib/sidekiq/fetch.rb

@@ -59,9 +59,9 @@ module Sidekiq
       end
 
       Sidekiq.redis do |conn|
-        conn.pipelined do
+        conn.pipelined do |pipeline|
           jobs_to_requeue.each do |queue, jobs|
-            conn.rpush(queue, jobs)
+            pipeline.rpush(queue, jobs)
           end
         end
       end

data/lib/sidekiq/job_logger.rb

@@ -12,46 +12,34 @@ module Sidekiq
 
       yield
 
-      with_elapsed_time_context(start) do
-        @logger.info("done")
-      end
+      Sidekiq::Context.add(:elapsed, elapsed(start))
+      @logger.info("done")
     rescue Exception
-      with_elapsed_time_context(start) do
-        @logger.info("fail")
-      end
+      Sidekiq::Context.add(:elapsed, elapsed(start))
+      @logger.info("fail")
 
       raise
     end
 
     def prepare(job_hash, &block)
-      level = job_hash["log_level"]
-      if level
-        @logger.log_at(level) do
-          Sidekiq::Context.with(job_hash_context(job_hash), &block)
-        end
-      else
-        Sidekiq::Context.with(job_hash_context(job_hash), &block)
-      end
-    end
-
-    def job_hash_context(job_hash)
       # If we're using a wrapper class, like ActiveJob, use the "wrapped"
       # attribute to expose the underlying thing.
       h = {
         class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
         jid: job_hash["jid"]
       }
-      h[:bid] = job_hash["bid"] if job_hash["bid"]
-      h[:tags] = job_hash["tags"] if job_hash["tags"]
-      h
-    end
-
-    def with_elapsed_time_context(start, &block)
-      Sidekiq::Context.with(elapsed_time_context(start), &block)
-    end
+      h[:bid] = job_hash["bid"] if job_hash.has_key?("bid")
+      h[:tags] = job_hash["tags"] if job_hash.has_key?("tags")
 
-    def elapsed_time_context(start)
-      {elapsed: elapsed(start).to_s}
+      Thread.current[:sidekiq_context] = h
+      level = job_hash["log_level"]
+      if level
+        @logger.log_at(level, &block)
+      else
+        yield
+      end
+    ensure
+      Thread.current[:sidekiq_context] = nil
     end
 
     private

data/lib/sidekiq/job_retry.rb

@@ -25,18 +25,19 @@ module Sidekiq
   #
   # A job looks like:
   #
-  #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => true }
+  #     { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => true }
   #
   # The 'retry' option also accepts a number (in place of 'true'):
   #
-  #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'], 'retry' => 5 }
+  #     { 'class' => 'HardJob', 'args' => [1, 2, 'foo'], 'retry' => 5 }
   #
   # The job will be retried this number of times before giving up. (If simply
   # 'true', Sidekiq retries 25 times)
   #
-  # We'll add a bit more data to the job to support retries:
+  # Relevant options for job retries:
   #
-  #  * 'queue' - the queue to use
+  #  * 'queue' - the queue for the initial job
+  #  * 'retry_queue' - if job retries should be pushed to a different (e.g. lower priority) queue
   #  * 'retry_count' - number of times we've retried so far.
   #  * 'error_message' - the message from the exception
   #  * 'error_class' - the exception class
@@ -52,11 +53,12 @@ module Sidekiq
   #
   #   Sidekiq.options[:max_retries] = 7
   #
-  # or limit the number of retries for a particular worker with:
+  # or limit the number of retries for a particular job and send retries to
+  # a low priority queue with:
   #
-  #    class MyWorker
-  #      include Sidekiq::Worker
-  #      sidekiq_options :retry => 10
+  #    class MyJob
+  #      include Sidekiq::Job
+  #      sidekiq_options retry: 10, retry_queue: 'low'
   #    end
   #
   class JobRetry
@@ -74,7 +76,7 @@ module Sidekiq
 
     # The global retry handler requires only the barest of data.
     # We want to be able to retry as much as possible so we don't
-    # require the worker to be instantiated.
+    # require the job to be instantiated.
     def global(jobstr, queue)
       yield
     rescue Handled => ex
@@ -101,14 +103,14 @@ module Sidekiq
     end
 
     # The local retry support means that any errors that occur within
-    # this block can be associated with the given worker instance.
+    # this block can be associated with the given job instance.
     # This is required to support the `sidekiq_retries_exhausted` block.
     #
     # Note that any exception from the block is wrapped in the Skip
     # exception so the global block does not reprocess the error.  The
     # Skip exception is unwrapped within Sidekiq::Processor#process before
     # calling the handle_exception handlers.
-    def local(worker, jobstr, queue)
+    def local(jobinst, jobstr, queue)
       yield
     rescue Handled => ex
       raise ex
@@ -121,11 +123,11 @@ module Sidekiq
 
       msg = Sidekiq.load_json(jobstr)
       if msg["retry"].nil?
-        msg["retry"] = worker.class.get_sidekiq_options["retry"]
+        msg["retry"] = jobinst.class.get_sidekiq_options["retry"]
       end
 
       raise e unless msg["retry"]
-      attempt_retry(worker, msg, queue, e)
+      attempt_retry(jobinst, msg, queue, e)
       # We've handled this error associated with this job, don't
       # need to handle it at the global level
       raise Skip
@@ -133,10 +135,10 @@ module Sidekiq
 
     private
 
-    # Note that +worker+ can be nil here if an error is raised before we can
-    # instantiate the worker instance.  All access must be guarded and
+    # Note that +jobinst+ can be nil here if an error is raised before we can
+    # instantiate the job instance.  All access must be guarded and
     # best effort.
-    def attempt_retry(worker, msg, queue, exception)
+    def attempt_retry(jobinst, msg, queue, exception)
       max_retry_attempts = retry_attempts_from(msg["retry"], @max_retries)
 
       msg["queue"] = (msg["retry_queue"] || queue)
@@ -168,7 +170,7 @@ module Sidekiq
       end
 
       if count < max_retry_attempts
-        delay = delay_for(worker, count, exception)
+        delay = delay_for(jobinst, count, exception)
         # Logging here can break retries if the logging device raises ENOSPC #3979
         # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
         retry_at = Time.now.to_f + delay
@@ -178,13 +180,13 @@ module Sidekiq
         end
       else
         # Goodbye dear message, you (re)tried your best I'm sure.
-        retries_exhausted(worker, msg, exception)
+        retries_exhausted(jobinst, msg, exception)
       end
     end
 
-    def retries_exhausted(worker, msg, exception)
+    def retries_exhausted(jobinst, msg, exception)
       begin
-        block = worker&.sidekiq_retries_exhausted_block
+        block = jobinst&.sidekiq_retries_exhausted_block
         block&.call(msg, exception)
       rescue => e
         handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
@@ -213,19 +215,19 @@ module Sidekiq
       end
     end
 
-    def delay_for(worker, count, exception)
+    def delay_for(jobinst, count, exception)
       jitter = rand(10) * (count + 1)
-      if worker&.sidekiq_retry_in_block
-        custom_retry_in = retry_in(worker, count, exception).to_i
+      if jobinst&.sidekiq_retry_in_block
+        custom_retry_in = retry_in(jobinst, count, exception).to_i
         return custom_retry_in + jitter if custom_retry_in > 0
       end
       (count**4) + 15 + jitter
     end
 
-    def retry_in(worker, count, exception)
-      worker.sidekiq_retry_in_block.call(count, exception)
+    def retry_in(jobinst, count, exception)
+      jobinst.sidekiq_retry_in_block.call(count, exception)
     rescue Exception => e
-      handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default"})
+      handle_exception(e, {context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{jobinst.class.name}, falling back to default"})
       nil
     end
 

data/lib/sidekiq/job_util.rb

@@ -0,0 +1,67 @@
+require "securerandom"
+require "time"
+
+module Sidekiq
+  module JobUtil
+    # These functions encapsulate various job utilities.
+    # They must be simple and free from side effects.
+
+    def validate(item)
+      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
+      raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
+      raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
+      raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
+      raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
+    end
+
+    def verify_json(item)
+      job_class = item["wrapped"] || item["class"]
+      if Sidekiq.options[:on_complex_arguments] == :raise
+        msg = <<~EOM
+          Job arguments to #{job_class} must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices.
+          To disable this error, remove `Sidekiq.strict_args!` from your initializer.
+        EOM
+        raise(ArgumentError, msg) unless json_safe?(item)
+      elsif Sidekiq.options[:on_complex_arguments] == :warn
+        Sidekiq.logger.warn <<~EOM unless json_safe?(item)
+          Job arguments to #{job_class} do not serialize to JSON safely. This will raise an error in
+          Sidekiq 7.0. See https://github.com/mperham/sidekiq/wiki/Best-Practices or raise an error today
+          by calling `Sidekiq.strict_args!` during Sidekiq initialization.
+        EOM
+      end
+    end
+
+    def normalize_item(item)
+      validate(item)
+
+      # merge in the default sidekiq_options for the item's class and/or wrapped element
+      # this allows ActiveJobs to control sidekiq_options too.
+      defaults = normalized_hash(item["class"])
+      defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?(:get_sidekiq_options)
+      item = defaults.merge(item)
+
+      raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
+
+      item["jid"] ||= SecureRandom.hex(12)
+      item["class"] = item["class"].to_s
+      item["queue"] = item["queue"].to_s
+      item["created_at"] ||= Time.now.to_f
+      item
+    end
+
+    def normalized_hash(item_class)
+      if item_class.is_a?(Class)
+        raise(ArgumentError, "Message must include a Sidekiq::Job class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?(:get_sidekiq_options)
+        item_class.get_sidekiq_options
+      else
+        Sidekiq.default_job_options
+      end
+    end
+
+    private
+
+    def json_safe?(item)
+      JSON.parse(JSON.dump(item["args"])) == item["args"]
+    end
+  end
+end