sidekiq 6.3.1 → 7.0.7

data/lib/sidekiq/client.rb

@@ -2,9 +2,12 @@
 
 require "securerandom"
 require "sidekiq/middleware/chain"
+require "sidekiq/job_util"
 
 module Sidekiq
   class Client
+    include Sidekiq::JobUtil
+
     ##
     # Define client-side middleware:
     #
@@ -12,13 +15,12 @@ module Sidekiq
     #   client.middleware do |chain|
     #     chain.use MyClientMiddleware
     #   end
-    #   client.push('class' => 'SomeWorker', 'args' => [1,2,3])
+    #   client.push('class' => 'SomeJob', 'args' => [1,2,3])
     #
     # All client instances default to the globally-defined
     # Sidekiq.client_middleware but you can change as necessary.
     #
     def middleware(&block)
-      @chain ||= Sidekiq.client_middleware
       if block
         @chain = @chain.dup
         yield @chain
@@ -28,34 +30,48 @@ module Sidekiq
 
     attr_accessor :redis_pool
 
-    # Sidekiq::Client normally uses the default Redis pool but you may
-    # pass a custom ConnectionPool if you want to shard your
-    # Sidekiq jobs across several Redis instances (for scalability
-    # reasons, e.g.)
+    # Sidekiq::Client is responsible for pushing job payloads to Redis.
+    # Requires the :pool or :config keyword argument.
     #
-    #   Sidekiq::Client.new(ConnectionPool.new { Redis.new })
+    #   Sidekiq::Client.new(pool: Sidekiq::RedisConnection.create)
     #
-    # Generally this is only needed for very large Sidekiq installs processing
-    # thousands of jobs per second.  I don't recommend sharding unless you
-    # cannot scale any other way (e.g. splitting your app into smaller apps).
-    def initialize(redis_pool = nil)
-      @redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
+    # Inside the Sidekiq process, you can reuse the configured resources:
+    #
+    #   Sidekiq::Client.new(config: config)
+    #
+    # @param pool [ConnectionPool] explicit Redis pool to use
+    # @param config [Sidekiq::Config] use the pool and middleware from the given Sidekiq container
+    # @param chain [Sidekiq::Middleware::Chain] use the given middleware chain
+    def initialize(*args, **kwargs)
+      if args.size == 1 && kwargs.size == 0
+        warn "Sidekiq::Client.new(pool) is deprecated, please use Sidekiq::Client.new(pool: pool), #{caller(0..3)}"
+        # old calling method, accept 1 pool argument
+        @redis_pool = args[0]
+        @chain = Sidekiq.default_configuration.client_middleware
+        @config = Sidekiq.default_configuration
+      else
+        # new calling method: keyword arguments
+        @config = kwargs[:config] || Sidekiq.default_configuration
+        @redis_pool = kwargs[:pool] || Thread.current[:sidekiq_redis_pool] || @config&.redis_pool
+        @chain = kwargs[:chain] || @config&.client_middleware
+        raise ArgumentError, "No Redis pool available for Sidekiq::Client" unless @redis_pool
+      end
     end
 
     ##
     # The main method used to push a job to Redis.  Accepts a number of options:
     #
     #   queue - the named queue to use, default 'default'
-    #   class - the worker class to call, required
+    #   class - the job class to call, required
     #   args - an array of simple arguments to the perform method, must be JSON-serializable
     #   at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
     #   retry - whether to retry this job if it fails, default true or an integer number of retries
     #   backtrace - whether to save any error backtrace, default false
     #
     # If class is set to the class name, the jobs' options will be based on Sidekiq's default
-    # worker options. Otherwise, they will be based on the job class's options.
+    # job options. Otherwise, they will be based on the job class's options.
     #
-    # Any options valid for a worker class's sidekiq_options are also available here.
+    # Any options valid for a job class's sidekiq_options are also available here.
     #
     # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
     # symbols in 'args' will be converted to strings.  Note that +backtrace: true+ can take quite a bit of
@@ -64,13 +80,15 @@ module Sidekiq
     # Returns a unique Job ID.  If middleware stops the job, nil will be returned instead.
     #
     # Example:
-    #   push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
+    #   push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
     #
     def push(item)
       normed = normalize_item(item)
-      payload = process_single(item["class"], normed)
-
+      payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
+        normed
+      end
       if payload
+        verify_json(payload)
         raw_push([payload])
         payload["jid"]
       end
@@ -98,12 +116,17 @@ module Sidekiq
       raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
       raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
+      jid = items.delete("jid")
+      raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
+
       normed = normalize_item(items)
       payloads = args.map.with_index { |job_args, index|
-        copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
+        copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
         copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
-
-        result = process_single(items["class"], copy)
+        result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
+          verify_json(copy)
+          copy
+        end
         result || nil
       }.compact
 
@@ -116,8 +139,8 @@ module Sidekiq
     #
     #   pool = ConnectionPool.new { Redis.new }
     #   Sidekiq::Client.via(pool) do
-    #     SomeWorker.perform_async(1,2,3)
-    #     SomeOtherWorker.perform_async(1,2,3)
+    #     SomeJob.perform_async(1,2,3)
+    #     SomeOtherJob.perform_async(1,2,3)
     #   end
     #
     # Generally this is only needed for very large Sidekiq installs processing
@@ -125,11 +148,11 @@ module Sidekiq
     # you cannot scale any other way (e.g. splitting your app into smaller apps).
     def self.via(pool)
       raise ArgumentError, "No pool given" if pool.nil?
-      current_sidekiq_pool = Thread.current[:sidekiq_via_pool]
-      Thread.current[:sidekiq_via_pool] = pool
+      current_sidekiq_pool = Thread.current[:sidekiq_redis_pool]
+      Thread.current[:sidekiq_redis_pool] = pool
       yield
     ensure
-      Thread.current[:sidekiq_via_pool] = current_sidekiq_pool
+      Thread.current[:sidekiq_redis_pool] = current_sidekiq_pool
     end
 
     class << self
@@ -142,10 +165,10 @@ module Sidekiq
       end
 
       # Resque compatibility helpers.  Note all helpers
-      # should go through Worker#client_push.
+      # should go through Sidekiq::Job#client_push.
       #
       # Example usage:
-      #   Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
       #
       # Messages are enqueued to the 'default' queue.
       #
@@ -154,19 +177,19 @@ module Sidekiq
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to(queue, klass, *args)
         klass.client_push("queue" => queue, "class" => klass, "args" => args)
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to_in(queue, interval, klass, *args)
         int = interval.to_f
         now = Time.now.to_f
-        ts = (int < 1_000_000_000 ? now + int : int)
+        ts = ((int < 1_000_000_000) ? now + int : int)
 
         item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
         item.delete("at") if ts <= now
@@ -175,7 +198,7 @@ module Sidekiq
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_in(interval, klass, *args)
         klass.perform_in(interval, *args)
@@ -186,8 +209,23 @@ module Sidekiq
 
     def raw_push(payloads)
       @redis_pool.with do |conn|
-        conn.pipelined do
-          atomic_push(conn, payloads)
+        retryable = true
+        begin
+          conn.pipelined do |pipeline|
+            atomic_push(pipeline, payloads)
+          end
+        rescue RedisClient::Error => ex
+          # 2550 Failover can cause the server to become a replica, need
+          # to disconnect and reopen the socket to get back to the primary.
+          # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
+          # 4985 Use the same logic when a blocking command is force-unblocked
+          # The retry logic is copied from sidekiq.rb
+          if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
+            conn.close
+            retryable = false
+            retry
+          end
+          raise
         end
       end
       true
@@ -195,7 +233,7 @@ module Sidekiq
 
     def atomic_push(conn, payloads)
       if payloads.first.key?("at")
-        conn.zadd("schedule", payloads.map { |hash|
+        conn.zadd("schedule", payloads.flat_map { |hash|
           at = hash.delete("at").to_s
           [at, Sidekiq.dump_json(hash)]
         })
@@ -206,54 +244,9 @@ module Sidekiq
           entry["enqueued_at"] = now
           Sidekiq.dump_json(entry)
         }
-        conn.sadd("queues", queue)
+        conn.sadd("queues", [queue])
         conn.lpush("queue:#{queue}", to_push)
       end
     end
-
-    def process_single(worker_class, item)
-      queue = item["queue"]
-
-      middleware.invoke(worker_class, item, queue, @redis_pool) do
-        item
-      end
-    end
-
-    def validate(item)
-      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
-      raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
-      raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
-      raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
-      raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
-    end
-
-    def normalize_item(item)
-      validate(item)
-      # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
-
-      # merge in the default sidekiq_options for the item's class and/or wrapped element
-      # this allows ActiveJobs to control sidekiq_options too.
-      defaults = normalized_hash(item["class"])
-      defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
-      item = defaults.merge(item)
-
-      raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
-
-      item["class"] = item["class"].to_s
-      item["queue"] = item["queue"].to_s
-      item["jid"] ||= SecureRandom.hex(12)
-      item["created_at"] ||= Time.now.to_f
-
-      item
-    end
-
-    def normalized_hash(item_class)
-      if item_class.is_a?(Class)
-        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
-        item_class.get_sidekiq_options
-      else
-        Sidekiq.default_worker_options
-      end
-    end
   end
 end

data/lib/sidekiq/{util.rb → component.rb}

@@ -1,43 +1,10 @@
 # frozen_string_literal: true
 
-require "forwardable"
-require "socket"
-require "securerandom"
-require "sidekiq/exception_handler"
-
 module Sidekiq
   ##
-  # This module is part of Sidekiq core and not intended for extensions.
-  #
-
-  class RingBuffer
-    include Enumerable
-    extend Forwardable
-    def_delegators :@buf, :[], :each, :size
-
-    def initialize(size, default = 0)
-      @size = size
-      @buf = Array.new(size, default)
-      @index = 0
-    end
-
-    def <<(element)
-      @buf[@index % @size] = element
-      @index += 1
-      element
-    end
-
-    def buffer
-      @buf
-    end
-
-    def reset(default = 0)
-      @buf.fill(default)
-    end
-  end
-
-  module Util
-    include ExceptionHandler
+  # Sidekiq::Component assumes a config instance is available at @config
+  module Component # :nodoc:
+    attr_reader :config
 
     def watchdog(last_words)
       yield
@@ -54,11 +21,11 @@ module Sidekiq
     end
 
     def logger
-      Sidekiq.logger
+      config.logger
     end
 
     def redis(&block)
-      Sidekiq.redis(&block)
+      config.redis(&block)
     end
 
     def tid
@@ -77,11 +44,17 @@ module Sidekiq
       @@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
     end
 
+    def handle_exception(ex, ctx = {})
+      config.handle_exception(ex, ctx)
+    end
+
     def fire_event(event, options = {})
+      oneshot = options.fetch(:oneshot, true)
       reverse = options[:reverse]
       reraise = options[:reraise]
+      logger.debug("Firing #{event} event") if oneshot
 
-      arr = Sidekiq.options[:lifecycle_events][event]
+      arr = config[:lifecycle_events][event]
       arr.reverse! if reverse
       arr.each do |block|
         block.call
@@ -89,7 +62,7 @@ module Sidekiq
         handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
         raise ex if reraise
       end
-      arr.clear
+      arr.clear if oneshot # once we've fired an event, we never fire it again
     end
   end
 end

data/lib/sidekiq/config.rb

@@ -0,0 +1,270 @@
+require "forwardable"
+
+require "set"
+require "sidekiq/redis_connection"
+
+module Sidekiq
+  # Sidekiq::Config represents the global configuration for an instance of Sidekiq.
+  class Config
+    extend Forwardable
+
+    DEFAULTS = {
+      labels: Set.new,
+      require: ".",
+      environment: nil,
+      concurrency: 5,
+      timeout: 25,
+      poll_interval_average: nil,
+      average_scheduled_poll_interval: 5,
+      on_complex_arguments: :raise,
+      error_handlers: [],
+      death_handlers: [],
+      lifecycle_events: {
+        startup: [],
+        quiet: [],
+        shutdown: [],
+        # triggers when we fire the first heartbeat on startup OR repairing a network partition
+        heartbeat: [],
+        # triggers on EVERY heartbeat call, every 10 seconds
+        beat: []
+      },
+      dead_max_jobs: 10_000,
+      dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
+      reloader: proc { |&block| block.call }
+    }
+
+    ERROR_HANDLER = ->(ex, ctx) {
+      cfg = ctx[:_config] || Sidekiq.default_configuration
+      l = cfg.logger
+      l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
+      l.warn("#{ex.class.name}: #{ex.message}")
+      l.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
+    }
+
+    def initialize(options = {})
+      @options = DEFAULTS.merge(options)
+      @options[:error_handlers] << ERROR_HANDLER if @options[:error_handlers].empty?
+      @directory = {}
+      @redis_config = {}
+      @capsules = {}
+    end
+
+    def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!
+    attr_reader :capsules
+
+    # LEGACY: edits the default capsule
+    # config.concurrency = 5
+    def concurrency=(val)
+      default_capsule.concurrency = Integer(val)
+    end
+
+    def concurrency
+      default_capsule.concurrency
+    end
+
+    def total_concurrency
+      capsules.each_value.sum(&:concurrency)
+    end
+
+    # Edit the default capsule.
+    # config.queues = %w( high default low )                 # strict
+    # config.queues = %w( high,3 default,2 low,1 )           # weighted
+    # config.queues = %w( feature1,1 feature2,1 feature3,1 ) # random
+    #
+    # With weighted priority, queue will be checked first (weight / total) of the time.
+    # high will be checked first (3/6) or 50% of the time.
+    # I'd recommend setting weights between 1-10. Weights in the hundreds or thousands
+    # are ridiculous and unnecessarily expensive. You can get random queue ordering
+    # by explicitly setting all weights to 1.
+    def queues=(val)
+      default_capsule.queues = val
+    end
+
+    def queues
+      default_capsule.queues
+    end
+
+    def client_middleware
+      @client_chain ||= Sidekiq::Middleware::Chain.new(self)
+      yield @client_chain if block_given?
+      @client_chain
+    end
+
+    def server_middleware
+      @server_chain ||= Sidekiq::Middleware::Chain.new(self)
+      yield @server_chain if block_given?
+      @server_chain
+    end
+
+    def default_capsule(&block)
+      capsule("default", &block)
+    end
+
+    # register a new queue processing subsystem
+    def capsule(name)
+      nm = name.to_s
+      cap = @capsules.fetch(nm) do
+        cap = Sidekiq::Capsule.new(nm, self)
+        @capsules[nm] = cap
+      end
+      yield cap if block_given?
+      cap
+    end
+
+    # All capsules must use the same Redis configuration
+    def redis=(hash)
+      @redis_config = @redis_config.merge(hash)
+    end
+
+    def redis_pool
+      Thread.current[:sidekiq_redis_pool] || Thread.current[:sidekiq_capsule]&.redis_pool || local_redis_pool
+    end
+
+    private def local_redis_pool
+      # this is our internal client/housekeeping pool. each capsule has its
+      # own pool for executing threads.
+      @redis ||= new_redis_pool(5, "internal")
+    end
+
+    def new_redis_pool(size, name = "unset")
+      # connection pool is lazy, it will not create connections unless you actually need them
+      # so don't be skimpy!
+      RedisConnection.create({size: size, logger: logger, pool_name: name}.merge(@redis_config))
+    end
+
+    def redis_info
+      redis do |conn|
+        conn.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
+      rescue RedisClientAdapter::CommandError => ex
+        # 2850 return fake version when INFO command has (probably) been renamed
+        raise unless /unknown command/.match?(ex.message)
+        {
+          "redis_version" => "9.9.9",
+          "uptime_in_days" => "9999",
+          "connected_clients" => "9999",
+          "used_memory_human" => "9P",
+          "used_memory_peak_human" => "9P"
+        }.freeze
+      end
+    end
+
+    def redis
+      raise ArgumentError, "requires a block" unless block_given?
+      redis_pool.with do |conn|
+        retryable = true
+        begin
+          yield conn
+        rescue RedisClientAdapter::BaseError => ex
+          # 2550 Failover can cause the server to become a replica, need
+          # to disconnect and reopen the socket to get back to the primary.
+          # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
+          # 4985 Use the same logic when a blocking command is force-unblocked
+          # The same retry logic is also used in client.rb
+          if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
+            conn.close
+            retryable = false
+            retry
+          end
+          raise
+        end
+      end
+    end
+
+    # register global singletons which can be accessed elsewhere
+    def register(name, instance)
+      @directory[name] = instance
+    end
+
+    # find a singleton
+    def lookup(name, default_class = nil)
+      # JNDI is just a fancy name for a hash lookup
+      @directory.fetch(name) do |key|
+        return nil unless default_class
+        @directory[key] = default_class.new(self)
+      end
+    end
+
+    ##
+    # Death handlers are called when all retries for a job have been exhausted and
+    # the job dies.  It's the notification to your application
+    # that this job will not succeed without manual intervention.
+    #
+    # Sidekiq.configure_server do |config|
+    #   config.death_handlers << ->(job, ex) do
+    #   end
+    # end
+    def death_handlers
+      @options[:death_handlers]
+    end
+
+    # How frequently Redis should be checked by a random Sidekiq process for
+    # scheduled and retriable jobs. Each individual process will take turns by
+    # waiting some multiple of this value.
+    #
+    # See sidekiq/scheduled.rb for an in-depth explanation of this value
+    def average_scheduled_poll_interval=(interval)
+      @options[:average_scheduled_poll_interval] = interval
+    end
+
+    # Register a proc to handle any error which occurs within the Sidekiq process.
+    #
+    #   Sidekiq.configure_server do |config|
+    #     config.error_handlers << proc {|ex,ctx_hash| MyErrorService.notify(ex, ctx_hash) }
+    #   end
+    #
+    # The default error handler logs errors to @logger.
+    def error_handlers
+      @options[:error_handlers]
+    end
+
+    # Register a block to run at a point in the Sidekiq lifecycle.
+    # :startup, :quiet or :shutdown are valid events.
+    #
+    #   Sidekiq.configure_server do |config|
+    #     config.on(:shutdown) do
+    #       puts "Goodbye cruel world!"
+    #     end
+    #   end
+    def on(event, &block)
+      raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
+      raise ArgumentError, "Invalid event name: #{event}" unless @options[:lifecycle_events].key?(event)
+      @options[:lifecycle_events][event] << block
+    end
+
+    def logger
+      @logger ||= Sidekiq::Logger.new($stdout, level: :info).tap do |log|
+        log.level = Logger::INFO
+        log.formatter = if ENV["DYNO"]
+          Sidekiq::Logger::Formatters::WithoutTimestamp.new
+        else
+          Sidekiq::Logger::Formatters::Pretty.new
+        end
+      end
+    end
+
+    def logger=(logger)
+      if logger.nil?
+        self.logger.level = Logger::FATAL
+        return
+      end
+
+      @logger = logger
+    end
+
+    # INTERNAL USE ONLY
+    def handle_exception(ex, ctx = {})
+      if @options[:error_handlers].size == 0
+        p ["!!!!!", ex]
+      end
+      ctx[:_config] = self
+      @options[:error_handlers].each do |handler|
+        handler.call(ex, ctx)
+      rescue => e
+        l = logger
+        l.error "!!! ERROR HANDLER THREW AN ERROR !!!"
+        l.error e
+        l.error e.backtrace.join("\n") unless e.backtrace.nil?
+      end
+    end
+  end
+end