sidekiq 6.0.4 → 7.2.0

data/lib/sidekiq/client.rb

@@ -2,9 +2,12 @@
 
 require "securerandom"
 require "sidekiq/middleware/chain"
+require "sidekiq/job_util"
 
 module Sidekiq
   class Client
+    include Sidekiq::JobUtil
+
     ##
     # Define client-side middleware:
     #
@@ -12,14 +15,13 @@ module Sidekiq
     #   client.middleware do |chain|
     #     chain.use MyClientMiddleware
     #   end
-    #   client.push('class' => 'SomeWorker', 'args' => [1,2,3])
+    #   client.push('class' => 'SomeJob', 'args' => [1,2,3])
     #
     # All client instances default to the globally-defined
     # Sidekiq.client_middleware but you can change as necessary.
     #
     def middleware(&block)
-      @chain ||= Sidekiq.client_middleware
-      if block_given?
+      if block
         @chain = @chain.dup
         yield @chain
       end
@@ -28,49 +30,66 @@ module Sidekiq
 
     attr_accessor :redis_pool
 
-    # Sidekiq::Client normally uses the default Redis pool but you may
-    # pass a custom ConnectionPool if you want to shard your
-    # Sidekiq jobs across several Redis instances (for scalability
-    # reasons, e.g.)
+    # Sidekiq::Client is responsible for pushing job payloads to Redis.
+    # Requires the :pool or :config keyword argument.
     #
-    #   Sidekiq::Client.new(ConnectionPool.new { Redis.new })
+    #   Sidekiq::Client.new(pool: Sidekiq::RedisConnection.create)
     #
-    # Generally this is only needed for very large Sidekiq installs processing
-    # thousands of jobs per second.  I don't recommend sharding unless you
-    # cannot scale any other way (e.g. splitting your app into smaller apps).
-    def initialize(redis_pool = nil)
-      @redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
+    # Inside the Sidekiq process, you can reuse the configured resources:
+    #
+    #   Sidekiq::Client.new(config: config)
+    #
+    # @param pool [ConnectionPool] explicit Redis pool to use
+    # @param config [Sidekiq::Config] use the pool and middleware from the given Sidekiq container
+    # @param chain [Sidekiq::Middleware::Chain] use the given middleware chain
+    def initialize(*args, **kwargs)
+      if args.size == 1 && kwargs.size == 0
+        warn "Sidekiq::Client.new(pool) is deprecated, please use Sidekiq::Client.new(pool: pool), #{caller(0..3)}"
+        # old calling method, accept 1 pool argument
+        @redis_pool = args[0]
+        @chain = Sidekiq.default_configuration.client_middleware
+        @config = Sidekiq.default_configuration
+      else
+        # new calling method: keyword arguments
+        @config = kwargs[:config] || Sidekiq.default_configuration
+        @redis_pool = kwargs[:pool] || Thread.current[:sidekiq_redis_pool] || @config&.redis_pool
+        @chain = kwargs[:chain] || @config&.client_middleware
+        raise ArgumentError, "No Redis pool available for Sidekiq::Client" unless @redis_pool
+      end
     end
 
     ##
     # The main method used to push a job to Redis.  Accepts a number of options:
     #
     #   queue - the named queue to use, default 'default'
-    #   class - the worker class to call, required
+    #   class - the job class to call, required
     #   args - an array of simple arguments to the perform method, must be JSON-serializable
     #   at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
     #   retry - whether to retry this job if it fails, default true or an integer number of retries
+    #   retry_for - relative amount of time to retry this job if it fails, default nil
     #   backtrace - whether to save any error backtrace, default false
     #
     # If class is set to the class name, the jobs' options will be based on Sidekiq's default
-    # worker options. Otherwise, they will be based on the job class's options.
+    # job options. Otherwise, they will be based on the job class's options.
     #
-    # Any options valid for a worker class's sidekiq_options are also available here.
+    # Any options valid for a job class's sidekiq_options are also available here.
     #
-    # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
+    # All keys must be strings, not symbols.  NB: because we are serializing to JSON, all
     # symbols in 'args' will be converted to strings.  Note that +backtrace: true+ can take quite a bit of
     # space in Redis; a large volume of failing jobs can start Redis swapping if you aren't careful.
     #
     # Returns a unique Job ID.  If middleware stops the job, nil will be returned instead.
     #
     # Example:
-    #   push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
+    #   push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
     #
     def push(item)
       normed = normalize_item(item)
-      payload = process_single(item["class"], normed)
-
+      payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
+        normed
+      end
       if payload
+        verify_json(payload)
         raw_push([payload])
         payload["jid"]
       end
@@ -78,8 +97,9 @@ module Sidekiq
 
     ##
     # Push a large number of jobs to Redis. This method cuts out the redis
-    # network round trip latency.  I wouldn't recommend pushing more than
-    # 1000 per call but YMMV based on network quality, size of job args, etc.
+    # network round trip latency. It pushes jobs in batches if more than
+    # `:batch_size` (1000 by default) of jobs are passed. I wouldn't recommend making `:batch_size`
+    # larger than 1000 but YMMV based on network quality, size of job args, etc.
     # A large number of jobs can cause a bit of Redis command processing latency.
     #
     # Takes the same arguments as #push except that args is expected to be
@@ -87,27 +107,45 @@ module Sidekiq
     # is run through the client middleware pipeline and each job gets its own Job ID
     # as normal.
     #
-    # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
-    # than the number given if the middleware stopped processing for one or more jobs.
+    # Returns an array of the of pushed jobs' jids, may contain nils if any client middleware
+    # prevented a job push.
+    #
+    # Example (pushing jobs in batches):
+    #   push_bulk('class' => MyJob, 'args' => (1..100_000).to_a, batch_size: 1_000)
+    #
     def push_bulk(items)
-      arg = items["args"].first
-      return [] unless arg # no jobs to push
-      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless arg.is_a?(Array)
-
+      batch_size = items.delete(:batch_size) || items.delete("batch_size") || 1_000
+      args = items["args"]
       at = items.delete("at")
-      raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
+      raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
+      raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
+
+      jid = items.delete("jid")
+      raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
 
       normed = normalize_item(items)
-      payloads = items["args"].map.with_index { |args, index|
-        copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
-        copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
+      slice_index = 0
+      result = args.each_slice(batch_size).flat_map do |slice|
+        raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless slice.is_a?(Array) && slice.all?(Array)
+        break [] if slice.empty? # no jobs to push
 
-        result = process_single(items["class"], copy)
-        result || nil
-      }.compact
+        payloads = slice.map.with_index { |job_args, index|
+          copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
+          copy["at"] = (at.is_a?(Array) ? at[slice_index + index] : at) if at
+          result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
+            verify_json(copy)
+            copy
+          end
+          result || nil
+        }
+        slice_index += batch_size
+
+        to_push = payloads.compact
+        raw_push(to_push) unless to_push.empty?
+        payloads.map { |payload| payload&.[]("jid") }
+      end
 
-      raw_push(payloads) unless payloads.empty?
-      payloads.collect { |payload| payload["jid"] }
+      result.is_a?(Enumerator::Lazy) ? result.force : result
     end
 
     # Allows sharding of jobs across any number of Redis instances.  All jobs
@@ -115,8 +153,8 @@ module Sidekiq
     #
     #   pool = ConnectionPool.new { Redis.new }
     #   Sidekiq::Client.via(pool) do
-    #     SomeWorker.perform_async(1,2,3)
-    #     SomeOtherWorker.perform_async(1,2,3)
+    #     SomeJob.perform_async(1,2,3)
+    #     SomeOtherJob.perform_async(1,2,3)
     #   end
     #
     # Generally this is only needed for very large Sidekiq installs processing
@@ -124,11 +162,11 @@ module Sidekiq
     # you cannot scale any other way (e.g. splitting your app into smaller apps).
     def self.via(pool)
       raise ArgumentError, "No pool given" if pool.nil?
-      current_sidekiq_pool = Thread.current[:sidekiq_via_pool]
-      Thread.current[:sidekiq_via_pool] = pool
+      current_sidekiq_pool = Thread.current[:sidekiq_redis_pool]
+      Thread.current[:sidekiq_redis_pool] = pool
       yield
     ensure
-      Thread.current[:sidekiq_via_pool] = current_sidekiq_pool
+      Thread.current[:sidekiq_redis_pool] = current_sidekiq_pool
     end
 
     class << self
@@ -136,15 +174,15 @@ module Sidekiq
         new.push(item)
       end
 
-      def push_bulk(items)
-        new.push_bulk(items)
+      def push_bulk(...)
+        new.push_bulk(...)
       end
 
       # Resque compatibility helpers.  Note all helpers
-      # should go through Worker#client_push.
+      # should go through Sidekiq::Job#client_push.
       #
       # Example usage:
-      #   Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
       #
       # Messages are enqueued to the 'default' queue.
       #
@@ -153,19 +191,19 @@ module Sidekiq
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to(queue, klass, *args)
         klass.client_push("queue" => queue, "class" => klass, "args" => args)
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to_in(queue, interval, klass, *args)
         int = interval.to_f
         now = Time.now.to_f
-        ts = (int < 1_000_000_000 ? now + int : int)
+        ts = ((int < 1_000_000_000) ? now + int : int)
 
         item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
         item.delete("at") if ts <= now
@@ -174,7 +212,7 @@ module Sidekiq
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_in(interval, klass, *args)
         klass.perform_in(interval, *args)
@@ -185,8 +223,23 @@ module Sidekiq
 
     def raw_push(payloads)
       @redis_pool.with do |conn|
-        conn.multi do
-          atomic_push(conn, payloads)
+        retryable = true
+        begin
+          conn.pipelined do |pipeline|
+            atomic_push(pipeline, payloads)
+          end
+        rescue RedisClient::Error => ex
+          # 2550 Failover can cause the server to become a replica, need
+          # to disconnect and reopen the socket to get back to the primary.
+          # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
+          # 4985 Use the same logic when a blocking command is force-unblocked
+          # The retry logic is copied from sidekiq.rb
+          if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
+            conn.close
+            retryable = false
+            retry
+          end
+          raise
         end
       end
       true
@@ -194,8 +247,10 @@ module Sidekiq
 
     def atomic_push(conn, payloads)
       if payloads.first.key?("at")
-        conn.zadd("schedule", payloads.map { |hash|
+        conn.zadd("schedule", payloads.flat_map { |hash|
           at = hash.delete("at").to_s
+          # ActiveJob sets this but the job has not been enqueued yet
+          hash.delete("enqueued_at")
           [at, Sidekiq.dump_json(hash)]
         })
       else
@@ -205,52 +260,9 @@ module Sidekiq
           entry["enqueued_at"] = now
           Sidekiq.dump_json(entry)
         }
-        conn.sadd("queues", queue)
+        conn.sadd("queues", [queue])
         conn.lpush("queue:#{queue}", to_push)
       end
     end
-
-    def process_single(worker_class, item)
-      queue = item["queue"]
-
-      middleware.invoke(worker_class, item, queue, @redis_pool) do
-        item
-      end
-    end
-
-    def normalize_item(item)
-      # 6.0.0 push_bulk bug, #4321
-      # TODO Remove after a while...
-      item.delete("at") if item.key?("at") && item["at"].nil?
-
-      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
-      raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
-      raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
-      raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
-      raise(ArgumentError, "Job tags must be an Array") if item["tags"] && !item["tags"].is_a?(Array)
-      # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
-
-      # merge in the default sidekiq_options for the item's class and/or wrapped element
-      # this allows ActiveJobs to control sidekiq_options too.
-      defaults = normalized_hash(item["class"])
-      defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
-      item = defaults.merge(item)
-
-      item["class"] = item["class"].to_s
-      item["queue"] = item["queue"].to_s
-      item["jid"] ||= SecureRandom.hex(12)
-      item["created_at"] ||= Time.now.to_f
-
-      item
-    end
-
-    def normalized_hash(item_class)
-      if item_class.is_a?(Class)
-        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
-        item_class.get_sidekiq_options
-      else
-        Sidekiq.default_worker_options
-      end
-    end
   end
 end

data/lib/sidekiq/{util.rb → component.rb}

@@ -1,15 +1,10 @@
 # frozen_string_literal: true
 
-require "socket"
-require "securerandom"
-require "sidekiq/exception_handler"
-
 module Sidekiq
   ##
-  # This module is part of Sidekiq core and not intended for extensions.
-  #
-  module Util
-    include ExceptionHandler
+  # Sidekiq::Component assumes a config instance is available at @config
+  module Component # :nodoc:
+    attr_reader :config
 
     def watchdog(last_words)
       yield
@@ -20,17 +15,17 @@ module Sidekiq
 
     def safe_thread(name, &block)
       Thread.new do
-        Thread.current.name = name
+        Thread.current.name = "sidekiq.#{name}"
         watchdog(name, &block)
       end
     end
 
     def logger
-      Sidekiq.logger
+      config.logger
     end
 
     def redis(&block)
-      Sidekiq.redis(&block)
+      config.redis(&block)
     end
 
     def tid
@@ -49,11 +44,17 @@ module Sidekiq
       @@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
     end
 
+    def handle_exception(ex, ctx = {})
+      config.handle_exception(ex, ctx)
+    end
+
     def fire_event(event, options = {})
+      oneshot = options.fetch(:oneshot, true)
       reverse = options[:reverse]
       reraise = options[:reraise]
+      logger.debug("Firing #{event} event") if oneshot
 
-      arr = Sidekiq.options[:lifecycle_events][event]
+      arr = config[:lifecycle_events][event]
       arr.reverse! if reverse
       arr.each do |block|
         block.call
@@ -61,7 +62,7 @@ module Sidekiq
         handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
         raise ex if reraise
       end
-      arr.clear
+      arr.clear if oneshot # once we've fired an event, we never fire it again
     end
   end
 end