sidekiq 6.4.0 → 7.1.2

data/lib/sidekiq/client.rb

@@ -15,13 +15,12 @@ module Sidekiq
     #   client.middleware do |chain|
     #     chain.use MyClientMiddleware
     #   end
-    #   client.push('class' => 'SomeWorker', 'args' => [1,2,3])
+    #   client.push('class' => 'SomeJob', 'args' => [1,2,3])
     #
     # All client instances default to the globally-defined
     # Sidekiq.client_middleware but you can change as necessary.
     #
     def middleware(&block)
-      @chain ||= Sidekiq.client_middleware
       if block
         @chain = @chain.dup
         yield @chain
@@ -31,34 +30,48 @@ module Sidekiq
 
     attr_accessor :redis_pool
 
-    # Sidekiq::Client normally uses the default Redis pool but you may
-    # pass a custom ConnectionPool if you want to shard your
-    # Sidekiq jobs across several Redis instances (for scalability
-    # reasons, e.g.)
+    # Sidekiq::Client is responsible for pushing job payloads to Redis.
+    # Requires the :pool or :config keyword argument.
     #
-    #   Sidekiq::Client.new(ConnectionPool.new { Redis.new })
+    #   Sidekiq::Client.new(pool: Sidekiq::RedisConnection.create)
     #
-    # Generally this is only needed for very large Sidekiq installs processing
-    # thousands of jobs per second.  I don't recommend sharding unless you
-    # cannot scale any other way (e.g. splitting your app into smaller apps).
-    def initialize(redis_pool = nil)
-      @redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
+    # Inside the Sidekiq process, you can reuse the configured resources:
+    #
+    #   Sidekiq::Client.new(config: config)
+    #
+    # @param pool [ConnectionPool] explicit Redis pool to use
+    # @param config [Sidekiq::Config] use the pool and middleware from the given Sidekiq container
+    # @param chain [Sidekiq::Middleware::Chain] use the given middleware chain
+    def initialize(*args, **kwargs)
+      if args.size == 1 && kwargs.size == 0
+        warn "Sidekiq::Client.new(pool) is deprecated, please use Sidekiq::Client.new(pool: pool), #{caller(0..3)}"
+        # old calling method, accept 1 pool argument
+        @redis_pool = args[0]
+        @chain = Sidekiq.default_configuration.client_middleware
+        @config = Sidekiq.default_configuration
+      else
+        # new calling method: keyword arguments
+        @config = kwargs[:config] || Sidekiq.default_configuration
+        @redis_pool = kwargs[:pool] || Thread.current[:sidekiq_redis_pool] || @config&.redis_pool
+        @chain = kwargs[:chain] || @config&.client_middleware
+        raise ArgumentError, "No Redis pool available for Sidekiq::Client" unless @redis_pool
+      end
     end
 
     ##
     # The main method used to push a job to Redis.  Accepts a number of options:
     #
     #   queue - the named queue to use, default 'default'
-    #   class - the worker class to call, required
+    #   class - the job class to call, required
     #   args - an array of simple arguments to the perform method, must be JSON-serializable
     #   at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
     #   retry - whether to retry this job if it fails, default true or an integer number of retries
     #   backtrace - whether to save any error backtrace, default false
     #
     # If class is set to the class name, the jobs' options will be based on Sidekiq's default
-    # worker options. Otherwise, they will be based on the job class's options.
+    # job options. Otherwise, they will be based on the job class's options.
     #
-    # Any options valid for a worker class's sidekiq_options are also available here.
+    # Any options valid for a job class's sidekiq_options are also available here.
     #
     # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
     # symbols in 'args' will be converted to strings.  Note that +backtrace: true+ can take quite a bit of
@@ -67,13 +80,15 @@ module Sidekiq
     # Returns a unique Job ID.  If middleware stops the job, nil will be returned instead.
     #
     # Example:
-    #   push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
+    #   push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
     #
     def push(item)
       normed = normalize_item(item)
-      payload = process_single(item["class"], normed)
-
+      payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
+        normed
+      end
       if payload
+        verify_json(payload)
         raw_push([payload])
         payload["jid"]
       end
@@ -81,8 +96,9 @@ module Sidekiq
 
     ##
     # Push a large number of jobs to Redis. This method cuts out the redis
-    # network round trip latency.  I wouldn't recommend pushing more than
-    # 1000 per call but YMMV based on network quality, size of job args, etc.
+    # network round trip latency. It pushes jobs in batches if more than
+    # `:batch_size` (1000 by default) of jobs are passed. I wouldn't recommend making `:batch_size`
+    # larger than 1000 but YMMV based on network quality, size of job args, etc.
     # A large number of jobs can cause a bit of Redis command processing latency.
     #
     # Takes the same arguments as #push except that args is expected to be
@@ -90,28 +106,43 @@ module Sidekiq
     # is run through the client middleware pipeline and each job gets its own Job ID
     # as normal.
     #
-    # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
-    # than the number given if the middleware stopped processing for one or more jobs.
+    # Returns an array of the of pushed jobs' jids, may contain nils if any client middleware
+    # prevented a job push.
+    #
+    # Example (pushing jobs in batches):
+    #   push_bulk('class' => 'MyJob', 'args' => (1..100_000).to_a, batch_size: 1_000)
+    #
     def push_bulk(items)
+      batch_size = items.delete(:batch_size) || items.delete("batch_size") || 1_000
       args = items["args"]
-      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
-      return [] if args.empty? # no jobs to push
-
       at = items.delete("at")
       raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
       raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
+      jid = items.delete("jid")
+      raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
+
       normed = normalize_item(items)
-      payloads = args.map.with_index { |job_args, index|
-        copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
-        copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
+      result = args.each_slice(batch_size).flat_map do |slice|
+        raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless slice.is_a?(Array) && slice.all?(Array)
+        break [] if slice.empty? # no jobs to push
 
-        result = process_single(items["class"], copy)
-        result || nil
-      }.compact
+        payloads = slice.map.with_index { |job_args, index|
+          copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
+          copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
+          result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
+            verify_json(copy)
+            copy
+          end
+          result || nil
+        }
+
+        to_push = payloads.compact
+        raw_push(to_push) unless to_push.empty?
+        payloads.map { |payload| payload&.[]("jid") }
+      end
 
-      raw_push(payloads) unless payloads.empty?
-      payloads.collect { |payload| payload["jid"] }
+      result.is_a?(Enumerator::Lazy) ? result.force : result
     end
 
     # Allows sharding of jobs across any number of Redis instances.  All jobs
@@ -119,8 +150,8 @@ module Sidekiq
     #
     #   pool = ConnectionPool.new { Redis.new }
     #   Sidekiq::Client.via(pool) do
-    #     SomeWorker.perform_async(1,2,3)
-    #     SomeOtherWorker.perform_async(1,2,3)
+    #     SomeJob.perform_async(1,2,3)
+    #     SomeOtherJob.perform_async(1,2,3)
     #   end
     #
     # Generally this is only needed for very large Sidekiq installs processing
@@ -128,11 +159,11 @@ module Sidekiq
     # you cannot scale any other way (e.g. splitting your app into smaller apps).
     def self.via(pool)
       raise ArgumentError, "No pool given" if pool.nil?
-      current_sidekiq_pool = Thread.current[:sidekiq_via_pool]
-      Thread.current[:sidekiq_via_pool] = pool
+      current_sidekiq_pool = Thread.current[:sidekiq_redis_pool]
+      Thread.current[:sidekiq_redis_pool] = pool
       yield
     ensure
-      Thread.current[:sidekiq_via_pool] = current_sidekiq_pool
+      Thread.current[:sidekiq_redis_pool] = current_sidekiq_pool
     end
 
     class << self
@@ -140,15 +171,15 @@ module Sidekiq
         new.push(item)
       end
 
-      def push_bulk(items)
-        new.push_bulk(items)
+      def push_bulk(...)
+        new.push_bulk(...)
       end
 
       # Resque compatibility helpers.  Note all helpers
-      # should go through Worker#client_push.
+      # should go through Sidekiq::Job#client_push.
       #
       # Example usage:
-      #   Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
       #
       # Messages are enqueued to the 'default' queue.
       #
@@ -157,19 +188,19 @@ module Sidekiq
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to(queue, klass, *args)
         klass.client_push("queue" => queue, "class" => klass, "args" => args)
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to_in(queue, interval, klass, *args)
         int = interval.to_f
         now = Time.now.to_f
-        ts = (int < 1_000_000_000 ? now + int : int)
+        ts = ((int < 1_000_000_000) ? now + int : int)
 
         item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
         item.delete("at") if ts <= now
@@ -178,7 +209,7 @@ module Sidekiq
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_in(interval, klass, *args)
         klass.perform_in(interval, *args)
@@ -189,8 +220,23 @@ module Sidekiq
 
     def raw_push(payloads)
       @redis_pool.with do |conn|
-        conn.pipelined do
-          atomic_push(conn, payloads)
+        retryable = true
+        begin
+          conn.pipelined do |pipeline|
+            atomic_push(pipeline, payloads)
+          end
+        rescue RedisClient::Error => ex
+          # 2550 Failover can cause the server to become a replica, need
+          # to disconnect and reopen the socket to get back to the primary.
+          # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
+          # 4985 Use the same logic when a blocking command is force-unblocked
+          # The retry logic is copied from sidekiq.rb
+          if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
+            conn.close
+            retryable = false
+            retry
+          end
+          raise
         end
       end
       true
@@ -198,8 +244,10 @@ module Sidekiq
 
     def atomic_push(conn, payloads)
       if payloads.first.key?("at")
-        conn.zadd("schedule", payloads.map { |hash|
+        conn.zadd("schedule", payloads.flat_map { |hash|
           at = hash.delete("at").to_s
+          # ActiveJob sets this but the job has not been enqueued yet
+          hash.delete("enqueued_at")
           [at, Sidekiq.dump_json(hash)]
         })
       else
@@ -209,17 +257,9 @@ module Sidekiq
           entry["enqueued_at"] = now
           Sidekiq.dump_json(entry)
         }
-        conn.sadd("queues", queue)
+        conn.sadd("queues", [queue])
         conn.lpush("queue:#{queue}", to_push)
       end
     end
-
-    def process_single(worker_class, item)
-      queue = item["queue"]
-
-      middleware.invoke(worker_class, item, queue, @redis_pool) do
-        item
-      end
-    end
   end
 end

data/lib/sidekiq/component.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  ##
+  # Sidekiq::Component assumes a config instance is available at @config
+  module Component # :nodoc:
+    attr_reader :config
+
+    def watchdog(last_words)
+      yield
+    rescue Exception => ex
+      handle_exception(ex, {context: last_words})
+      raise ex
+    end
+
+    def safe_thread(name, &block)
+      Thread.new do
+        Thread.current.name = "sidekiq.#{name}"
+        watchdog(name, &block)
+      end
+    end
+
+    def logger
+      config.logger
+    end
+
+    def redis(&block)
+      config.redis(&block)
+    end
+
+    def tid
+      Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
+    end
+
+    def hostname
+      ENV["DYNO"] || Socket.gethostname
+    end
+
+    def process_nonce
+      @@process_nonce ||= SecureRandom.hex(6)
+    end
+
+    def identity
+      @@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
+    end
+
+    def handle_exception(ex, ctx = {})
+      config.handle_exception(ex, ctx)
+    end
+
+    def fire_event(event, options = {})
+      oneshot = options.fetch(:oneshot, true)
+      reverse = options[:reverse]
+      reraise = options[:reraise]
+      logger.debug("Firing #{event} event") if oneshot
+
+      arr = config[:lifecycle_events][event]
+      arr.reverse! if reverse
+      arr.each do |block|
+        block.call
+      rescue => ex
+        handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
+        raise ex if reraise
+      end
+      arr.clear if oneshot # once we've fired an event, we never fire it again
+    end
+  end
+end

data/lib/sidekiq/config.rb

@@ -0,0 +1,278 @@
+require "forwardable"
+
+require "set"
+require "sidekiq/redis_connection"
+
+module Sidekiq
+  # Sidekiq::Config represents the global configuration for an instance of Sidekiq.
+  class Config
+    extend Forwardable
+
+    DEFAULTS = {
+      labels: Set.new,
+      require: ".",
+      environment: nil,
+      concurrency: 5,
+      timeout: 25,
+      poll_interval_average: nil,
+      average_scheduled_poll_interval: 5,
+      on_complex_arguments: :raise,
+      error_handlers: [],
+      death_handlers: [],
+      lifecycle_events: {
+        startup: [],
+        quiet: [],
+        shutdown: [],
+        # triggers when we fire the first heartbeat on startup OR repairing a network partition
+        heartbeat: [],
+        # triggers on EVERY heartbeat call, every 10 seconds
+        beat: []
+      },
+      dead_max_jobs: 10_000,
+      dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
+      reloader: proc { |&block| block.call },
+      backtrace_cleaner: ->(backtrace) { backtrace }
+    }
+
+    ERROR_HANDLER = ->(ex, ctx) {
+      cfg = ctx[:_config] || Sidekiq.default_configuration
+      l = cfg.logger
+      l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
+      l.warn("#{ex.class.name}: #{ex.message}")
+      unless ex.backtrace.nil?
+        backtrace = cfg[:backtrace_cleaner].call(ex.backtrace)
+        l.warn(backtrace.join("\n"))
+      end
+    }
+
+    def initialize(options = {})
+      @options = DEFAULTS.merge(options)
+      @options[:error_handlers] << ERROR_HANDLER if @options[:error_handlers].empty?
+      @directory = {}
+      @redis_config = {}
+      @capsules = {}
+    end
+
+    def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!
+    attr_reader :capsules
+
+    def to_json(*)
+      Sidekiq.dump_json(@options)
+    end
+
+    # LEGACY: edits the default capsule
+    # config.concurrency = 5
+    def concurrency=(val)
+      default_capsule.concurrency = Integer(val)
+    end
+
+    def concurrency
+      default_capsule.concurrency
+    end
+
+    def total_concurrency
+      capsules.each_value.sum(&:concurrency)
+    end
+
+    # Edit the default capsule.
+    # config.queues = %w( high default low )                 # strict
+    # config.queues = %w( high,3 default,2 low,1 )           # weighted
+    # config.queues = %w( feature1,1 feature2,1 feature3,1 ) # random
+    #
+    # With weighted priority, queue will be checked first (weight / total) of the time.
+    # high will be checked first (3/6) or 50% of the time.
+    # I'd recommend setting weights between 1-10. Weights in the hundreds or thousands
+    # are ridiculous and unnecessarily expensive. You can get random queue ordering
+    # by explicitly setting all weights to 1.
+    def queues=(val)
+      default_capsule.queues = val
+    end
+
+    def queues
+      default_capsule.queues
+    end
+
+    def client_middleware
+      @client_chain ||= Sidekiq::Middleware::Chain.new(self)
+      yield @client_chain if block_given?
+      @client_chain
+    end
+
+    def server_middleware
+      @server_chain ||= Sidekiq::Middleware::Chain.new(self)
+      yield @server_chain if block_given?
+      @server_chain
+    end
+
+    def default_capsule(&block)
+      capsule("default", &block)
+    end
+
+    # register a new queue processing subsystem
+    def capsule(name)
+      nm = name.to_s
+      cap = @capsules.fetch(nm) do
+        cap = Sidekiq::Capsule.new(nm, self)
+        @capsules[nm] = cap
+      end
+      yield cap if block_given?
+      cap
+    end
+
+    # All capsules must use the same Redis configuration
+    def redis=(hash)
+      @redis_config = @redis_config.merge(hash)
+    end
+
+    def redis_pool
+      Thread.current[:sidekiq_redis_pool] || Thread.current[:sidekiq_capsule]&.redis_pool || local_redis_pool
+    end
+
+    private def local_redis_pool
+      # this is our internal client/housekeeping pool. each capsule has its
+      # own pool for executing threads.
+      @redis ||= new_redis_pool(10, "internal")
+    end
+
+    def new_redis_pool(size, name = "unset")
+      # connection pool is lazy, it will not create connections unless you actually need them
+      # so don't be skimpy!
+      RedisConnection.create({size: size, logger: logger, pool_name: name}.merge(@redis_config))
+    end
+
+    def redis_info
+      redis do |conn|
+        conn.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
+      rescue RedisClientAdapter::CommandError => ex
+        # 2850 return fake version when INFO command has (probably) been renamed
+        raise unless /unknown command/.match?(ex.message)
+        {
+          "redis_version" => "9.9.9",
+          "uptime_in_days" => "9999",
+          "connected_clients" => "9999",
+          "used_memory_human" => "9P",
+          "used_memory_peak_human" => "9P"
+        }.freeze
+      end
+    end
+
+    def redis
+      raise ArgumentError, "requires a block" unless block_given?
+      redis_pool.with do |conn|
+        retryable = true
+        begin
+          yield conn
+        rescue RedisClientAdapter::BaseError => ex
+          # 2550 Failover can cause the server to become a replica, need
+          # to disconnect and reopen the socket to get back to the primary.
+          # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
+          # 4985 Use the same logic when a blocking command is force-unblocked
+          # The same retry logic is also used in client.rb
+          if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
+            conn.close
+            retryable = false
+            retry
+          end
+          raise
+        end
+      end
+    end
+
+    # register global singletons which can be accessed elsewhere
+    def register(name, instance)
+      @directory[name] = instance
+    end
+
+    # find a singleton
+    def lookup(name, default_class = nil)
+      # JNDI is just a fancy name for a hash lookup
+      @directory.fetch(name) do |key|
+        return nil unless default_class
+        @directory[key] = default_class.new(self)
+      end
+    end
+
+    ##
+    # Death handlers are called when all retries for a job have been exhausted and
+    # the job dies.  It's the notification to your application
+    # that this job will not succeed without manual intervention.
+    #
+    # Sidekiq.configure_server do |config|
+    #   config.death_handlers << ->(job, ex) do
+    #   end
+    # end
+    def death_handlers
+      @options[:death_handlers]
+    end
+
+    # How frequently Redis should be checked by a random Sidekiq process for
+    # scheduled and retriable jobs. Each individual process will take turns by
+    # waiting some multiple of this value.
+    #
+    # See sidekiq/scheduled.rb for an in-depth explanation of this value
+    def average_scheduled_poll_interval=(interval)
+      @options[:average_scheduled_poll_interval] = interval
+    end
+
+    # Register a proc to handle any error which occurs within the Sidekiq process.
+    #
+    #   Sidekiq.configure_server do |config|
+    #     config.error_handlers << proc {|ex,ctx_hash| MyErrorService.notify(ex, ctx_hash) }
+    #   end
+    #
+    # The default error handler logs errors to @logger.
+    def error_handlers
+      @options[:error_handlers]
+    end
+
+    # Register a block to run at a point in the Sidekiq lifecycle.
+    # :startup, :quiet or :shutdown are valid events.
+    #
+    #   Sidekiq.configure_server do |config|
+    #     config.on(:shutdown) do
+    #       puts "Goodbye cruel world!"
+    #     end
+    #   end
+    def on(event, &block)
+      raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
+      raise ArgumentError, "Invalid event name: #{event}" unless @options[:lifecycle_events].key?(event)
+      @options[:lifecycle_events][event] << block
+    end
+
+    def logger
+      @logger ||= Sidekiq::Logger.new($stdout, level: :info).tap do |log|
+        log.level = Logger::INFO
+        log.formatter = if ENV["DYNO"]
+          Sidekiq::Logger::Formatters::WithoutTimestamp.new
+        else
+          Sidekiq::Logger::Formatters::Pretty.new
+        end
+      end
+    end
+
+    def logger=(logger)
+      if logger.nil?
+        self.logger.level = Logger::FATAL
+        return
+      end
+
+      @logger = logger
+    end
+
+    # INTERNAL USE ONLY
+    def handle_exception(ex, ctx = {})
+      if @options[:error_handlers].size == 0
+        p ["!!!!!", ex]
+      end
+      ctx[:_config] = self
+      @options[:error_handlers].each do |handler|
+        handler.call(ex, ctx)
+      rescue Exception => e
+        l = logger
+        l.error "!!! ERROR HANDLER THREW AN ERROR !!!"
+        l.error e
+        l.error e.backtrace.join("\n") unless e.backtrace.nil?
+      end
+    end
+  end
+end