sidekiq 3.4.1 → 7.3.0

data/lib/sidekiq/config.rb

@@ -0,0 +1,291 @@
+require "forwardable"
+
+require "set"
+require "sidekiq/redis_connection"
+
+module Sidekiq
+  # Sidekiq::Config represents the global configuration for an instance of Sidekiq.
+  class Config
+    extend Forwardable
+
+    DEFAULTS = {
+      labels: Set.new,
+      require: ".",
+      environment: nil,
+      concurrency: 5,
+      timeout: 25,
+      poll_interval_average: nil,
+      average_scheduled_poll_interval: 5,
+      on_complex_arguments: :raise,
+      iteration: {
+        max_job_runtime: nil,
+        retry_backoff: 0
+      },
+      error_handlers: [],
+      death_handlers: [],
+      lifecycle_events: {
+        startup: [],
+        quiet: [],
+        shutdown: [],
+        # triggers when we fire the first heartbeat on startup OR repairing a network partition
+        heartbeat: [],
+        # triggers on EVERY heartbeat call, every 10 seconds
+        beat: []
+      },
+      dead_max_jobs: 10_000,
+      dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
+      reloader: proc { |&block| block.call },
+      backtrace_cleaner: ->(backtrace) { backtrace }
+    }
+
+    ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
+      l = cfg.logger
+      l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
+      l.warn("#{ex.class.name}: #{ex.message}")
+      unless ex.backtrace.nil?
+        backtrace = cfg[:backtrace_cleaner].call(ex.backtrace)
+        l.warn(backtrace.join("\n"))
+      end
+    }
+
+    def initialize(options = {})
+      @options = DEFAULTS.merge(options)
+      @options[:error_handlers] << ERROR_HANDLER if @options[:error_handlers].empty?
+      @directory = {}
+      @redis_config = {}
+      @capsules = {}
+    end
+
+    def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
+    attr_reader :capsules
+
+    def to_json(*)
+      Sidekiq.dump_json(@options)
+    end
+
+    # LEGACY: edits the default capsule
+    # config.concurrency = 5
+    def concurrency=(val)
+      default_capsule.concurrency = Integer(val)
+    end
+
+    def concurrency
+      default_capsule.concurrency
+    end
+
+    def total_concurrency
+      capsules.each_value.sum(&:concurrency)
+    end
+
+    # Edit the default capsule.
+    # config.queues = %w( high default low )                 # strict
+    # config.queues = %w( high,3 default,2 low,1 )           # weighted
+    # config.queues = %w( feature1,1 feature2,1 feature3,1 ) # random
+    #
+    # With weighted priority, queue will be checked first (weight / total) of the time.
+    # high will be checked first (3/6) or 50% of the time.
+    # I'd recommend setting weights between 1-10. Weights in the hundreds or thousands
+    # are ridiculous and unnecessarily expensive. You can get random queue ordering
+    # by explicitly setting all weights to 1.
+    def queues=(val)
+      default_capsule.queues = val
+    end
+
+    def queues
+      default_capsule.queues
+    end
+
+    def client_middleware
+      @client_chain ||= Sidekiq::Middleware::Chain.new(self)
+      yield @client_chain if block_given?
+      @client_chain
+    end
+
+    def server_middleware
+      @server_chain ||= Sidekiq::Middleware::Chain.new(self)
+      yield @server_chain if block_given?
+      @server_chain
+    end
+
+    def default_capsule(&block)
+      capsule("default", &block)
+    end
+
+    # register a new queue processing subsystem
+    def capsule(name)
+      nm = name.to_s
+      cap = @capsules.fetch(nm) do
+        cap = Sidekiq::Capsule.new(nm, self)
+        @capsules[nm] = cap
+      end
+      yield cap if block_given?
+      cap
+    end
+
+    # All capsules must use the same Redis configuration
+    def redis=(hash)
+      @redis_config = @redis_config.merge(hash)
+    end
+
+    def redis_pool
+      Thread.current[:sidekiq_redis_pool] || Thread.current[:sidekiq_capsule]&.redis_pool || local_redis_pool
+    end
+
+    private def local_redis_pool
+      # this is our internal client/housekeeping pool. each capsule has its
+      # own pool for executing threads.
+      @redis ||= new_redis_pool(10, "internal")
+    end
+
+    def new_redis_pool(size, name = "unset")
+      # connection pool is lazy, it will not create connections unless you actually need them
+      # so don't be skimpy!
+      RedisConnection.create({size: size, logger: logger, pool_name: name}.merge(@redis_config))
+    end
+
+    def redis_info
+      redis do |conn|
+        conn.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
+      rescue RedisClientAdapter::CommandError => ex
+        # 2850 return fake version when INFO command has (probably) been renamed
+        raise unless /unknown command/.match?(ex.message)
+        {
+          "redis_version" => "9.9.9",
+          "uptime_in_days" => "9999",
+          "connected_clients" => "9999",
+          "used_memory_human" => "9P",
+          "used_memory_peak_human" => "9P"
+        }.freeze
+      end
+    end
+
+    def redis
+      raise ArgumentError, "requires a block" unless block_given?
+      redis_pool.with do |conn|
+        retryable = true
+        begin
+          yield conn
+        rescue RedisClientAdapter::BaseError => ex
+          # 2550 Failover can cause the server to become a replica, need
+          # to disconnect and reopen the socket to get back to the primary.
+          # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
+          # 4985 Use the same logic when a blocking command is force-unblocked
+          # The same retry logic is also used in client.rb
+          if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
+            conn.close
+            retryable = false
+            retry
+          end
+          raise
+        end
+      end
+    end
+
+    # register global singletons which can be accessed elsewhere
+    def register(name, instance)
+      @directory[name] = instance
+    end
+
+    # find a singleton
+    def lookup(name, default_class = nil)
+      # JNDI is just a fancy name for a hash lookup
+      @directory.fetch(name) do |key|
+        return nil unless default_class
+        @directory[key] = default_class.new(self)
+      end
+    end
+
+    ##
+    # Death handlers are called when all retries for a job have been exhausted and
+    # the job dies.  It's the notification to your application
+    # that this job will not succeed without manual intervention.
+    #
+    # Sidekiq.configure_server do |config|
+    #   config.death_handlers << ->(job, ex) do
+    #   end
+    # end
+    def death_handlers
+      @options[:death_handlers]
+    end
+
+    # How frequently Redis should be checked by a random Sidekiq process for
+    # scheduled and retriable jobs. Each individual process will take turns by
+    # waiting some multiple of this value.
+    #
+    # See sidekiq/scheduled.rb for an in-depth explanation of this value
+    def average_scheduled_poll_interval=(interval)
+      @options[:average_scheduled_poll_interval] = interval
+    end
+
+    # Register a proc to handle any error which occurs within the Sidekiq process.
+    #
+    #   Sidekiq.configure_server do |config|
+    #     config.error_handlers << proc {|ex,ctx_hash| MyErrorService.notify(ex, ctx_hash) }
+    #   end
+    #
+    # The default error handler logs errors to @logger.
+    def error_handlers
+      @options[:error_handlers]
+    end
+
+    # Register a block to run at a point in the Sidekiq lifecycle.
+    # :startup, :quiet or :shutdown are valid events.
+    #
+    #   Sidekiq.configure_server do |config|
+    #     config.on(:shutdown) do
+    #       puts "Goodbye cruel world!"
+    #     end
+    #   end
+    def on(event, &block)
+      raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
+      raise ArgumentError, "Invalid event name: #{event}" unless @options[:lifecycle_events].key?(event)
+      @options[:lifecycle_events][event] << block
+    end
+
+    def logger
+      @logger ||= Sidekiq::Logger.new($stdout, level: :info).tap do |log|
+        log.level = Logger::INFO
+        log.formatter = if ENV["DYNO"]
+          Sidekiq::Logger::Formatters::WithoutTimestamp.new
+        else
+          Sidekiq::Logger::Formatters::Pretty.new
+        end
+      end
+    end
+
+    def logger=(logger)
+      if logger.nil?
+        self.logger.level = Logger::FATAL
+        return
+      end
+
+      @logger = logger
+    end
+
+    private def parameter_size(handler)
+      target = handler.is_a?(Proc) ? handler : handler.method(:call)
+      target.parameters.size
+    end
+
+    # INTERNAL USE ONLY
+    def handle_exception(ex, ctx = {})
+      if @options[:error_handlers].size == 0
+        p ["!!!!!", ex]
+      end
+      @options[:error_handlers].each do |handler|
+        if parameter_size(handler) == 2
+          # TODO Remove in 8.0
+          logger.info { "DEPRECATION: Sidekiq exception handlers now take three arguments, see #{handler}" }
+          handler.call(ex, {_config: self}.merge(ctx))
+        else
+          handler.call(ex, ctx, self)
+        end
+      rescue Exception => e
+        l = logger
+        l.error "!!! ERROR HANDLER THREW AN ERROR !!!"
+        l.error e
+        l.error e.backtrace.join("\n") unless e.backtrace.nil?
+      end
+    end
+  end
+end

data/lib/sidekiq/deploy.rb

@@ -0,0 +1,62 @@
+require "sidekiq/redis_connection"
+require "time"
+
+# This file is designed to be required within the user's
+# deployment script; it should need a bare minimum of dependencies.
+# Usage:
+#
+#   require "sidekiq/deploy"
+#   Sidekiq::Deploy.mark!("Some change")
+#
+# If you do not pass a label, Sidekiq will try to use the latest
+# git commit info.
+#
+
+module Sidekiq
+  class Deploy
+    MARK_TTL = 90 * 24 * 60 * 60 # 90 days
+
+    LABEL_MAKER = -> {
+      `git log -1 --format="%h %s"`.strip
+    }
+
+    def self.mark!(label = nil)
+      Sidekiq::Deploy.new.mark!(label: label)
+    end
+
+    def initialize(pool = Sidekiq::RedisConnection.create)
+      @pool = pool
+    end
+
+    def mark!(at: Time.now, label: nil)
+      label ||= LABEL_MAKER.call
+      # we need to round the timestamp so that we gracefully
+      # handle an very common error in marking deploys:
+      # having every process mark its deploy, leading
+      # to N marks for each deploy. Instead we round the time
+      # to the minute so that multiple marks within that minute
+      # will all naturally rollup into one mark per minute.
+      whence = at.utc
+      floor = Time.utc(whence.year, whence.month, whence.mday, whence.hour, whence.min, 0)
+      datecode = floor.strftime("%Y%m%d")
+      key = "#{datecode}-marks"
+      stamp = floor.iso8601
+
+      @pool.with do |c|
+        # only allow one deploy mark for a given label for the next minute
+        lock = c.set("deploylock-#{label}", stamp, "nx", "ex", "60")
+        if lock
+          c.multi do |pipe|
+            pipe.hsetnx(key, stamp, label)
+            pipe.expire(key, MARK_TTL)
+          end
+        end
+      end
+    end
+
+    def fetch(date = Time.now.utc.to_date)
+      datecode = date.strftime("%Y%m%d")
+      @pool.with { |c| c.hgetall("#{datecode}-marks") }
+    end
+  end
+end

data/lib/sidekiq/embedded.rb

@@ -0,0 +1,61 @@
+require "sidekiq/component"
+require "sidekiq/launcher"
+require "sidekiq/metrics/tracking"
+
+module Sidekiq
+  class Embedded
+    include Sidekiq::Component
+
+    def initialize(config)
+      @config = config
+    end
+
+    def run
+      housekeeping
+      fire_event(:startup, reverse: false, reraise: true)
+      @launcher = Sidekiq::Launcher.new(@config, embedded: true)
+      @launcher.run
+      sleep 0.2 # pause to give threads time to spin up
+
+      logger.info "Sidekiq running embedded, total process thread count: #{Thread.list.size}"
+      logger.debug { Thread.list.map(&:name) }
+    end
+
+    def quiet
+      @launcher&.quiet
+    end
+
+    def stop
+      @launcher&.stop
+    end
+
+    private
+
+    def housekeeping
+      logger.info "Running in #{RUBY_DESCRIPTION}"
+      logger.info Sidekiq::LICENSE
+      logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
+
+      # touch the connection pool so it is created before we
+      # fire startup and start multithreading.
+      info = config.redis_info
+      ver = Gem::Version.new(info["redis_version"])
+      raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
+
+      maxmemory_policy = info["maxmemory_policy"]
+      if maxmemory_policy != "noeviction"
+        logger.warn <<~EOM
+
+
+          WARNING: Your Redis instance will evict Sidekiq data under heavy load.
+          The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
+          See: https://github.com/sidekiq/sidekiq/wiki/Using-Redis#memory
+
+        EOM
+      end
+
+      logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
+      logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
+    end
+  end
+end

data/lib/sidekiq/fetch.rb

@@ -1,146 +1,73 @@
-require 'sidekiq'
-require 'sidekiq/util'
-require 'sidekiq/actor'
+# frozen_string_literal: true
 
-module Sidekiq
-  ##
-  # The Fetcher blocks on Redis, waiting for a message to process
-  # from the queues.  It gets the message and hands it to the Manager
-  # to assign to a ready Processor.
-  class Fetcher
-    include Util
-    include Actor
+require "sidekiq"
+require "sidekiq/component"
+require "sidekiq/capsule"
 
-    TIMEOUT = 1
-
-    attr_reader :down
-
-    def initialize(mgr, options)
-      @down = nil
-      @mgr = mgr
-      @strategy = Fetcher.strategy.new(options)
-    end
-
-    # Fetching is straightforward: the Manager makes a fetch
-    # request for each idle processor when Sidekiq starts and
-    # then issues a new fetch request every time a Processor
-    # finishes a message.
-    #
-    # Because we have to shut down cleanly, we can't block
-    # forever and we can't loop forever.  Instead we reschedule
-    # a new fetch if the current fetch turned up nothing.
-    def fetch
-      watchdog('Fetcher#fetch died') do
-        return if Sidekiq::Fetcher.done?
-
-        begin
-          work = @strategy.retrieve_work
-          ::Sidekiq.logger.info("Redis is online, #{Time.now - @down} sec downtime") if @down
-          @down = nil
-
-          if work
-            @mgr.async.assign(work)
-          else
-            after(0) { fetch }
-          end
-        rescue => ex
-          handle_fetch_exception(ex)
-        end
+module Sidekiq # :nodoc:
+  class BasicFetch
+    include Sidekiq::Component
+    # We want the fetch operation to timeout every few seconds so the thread
+    # can check if the process is shutting down.
+    TIMEOUT = 2
 
+    UnitOfWork = Struct.new(:queue, :job, :config) {
+      def acknowledge
+        # nothing to do
       end
-    end
-
-    private
 
-    def pause
-      sleep(TIMEOUT)
-    end
+      def queue_name
+        queue.delete_prefix("queue:")
+      end
 
-    def handle_fetch_exception(ex)
-      if !@down
-        logger.error("Error fetching message: #{ex}")
-        ex.backtrace.each do |bt|
-          logger.error(bt)
+      def requeue
+        config.redis do |conn|
+          conn.rpush(queue, job)
         end
       end
-      @down ||= Time.now
-      pause
-      after(0) { fetch }
-    rescue Task::TerminatedError
-      # If redis is down when we try to shut down, all the fetch backlog
-      # raises these errors.  Haven't been able to figure out what I'm doing wrong.
-    end
-
-    # Ugh.  Say hello to a bloody hack.
-    # Can't find a clean way to get the fetcher to just stop processing
-    # its mailbox when shutdown starts.
-    def self.done!
-      @done = true
-    end
-
-    def self.reset # testing only
-      @done = nil
-    end
-
-    def self.done?
-      @done
-    end
-
-    def self.strategy
-      Sidekiq.options[:fetch] || BasicFetch
-    end
-  end
-
-  class BasicFetch
-    def initialize(options)
-      @strictly_ordered_queues = !!options[:strict]
-      @queues = options[:queues].map { |q| "queue:#{q}" }
-      @unique_queues = @queues.uniq
+    }
+
+    def initialize(cap)
+      raise ArgumentError, "missing queue list" unless cap.queues
+      @config = cap
+      @strictly_ordered_queues = cap.mode == :strict
+      @queues = config.queues.map { |q| "queue:#{q}" }
+      @queues.uniq! if @strictly_ordered_queues
     end
 
     def retrieve_work
-      work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
-      UnitOfWork.new(*work) if work
+      qs = queues_cmd
+      # 4825 Sidekiq Pro with all queues paused will return an
+      # empty set of queues
+      if qs.size <= 0
+        sleep(TIMEOUT)
+        return nil
+      end
+
+      queue, job = redis { |conn| conn.blocking_call(TIMEOUT, "brpop", *qs, TIMEOUT) }
+      UnitOfWork.new(queue, job, config) if queue
     end
 
-    # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
-    # an instance method will make it async to the Fetcher actor
-    def self.bulk_requeue(inprogress, options)
+    def bulk_requeue(inprogress)
       return if inprogress.empty?
 
-      Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+      logger.debug { "Re-queueing terminated jobs" }
       jobs_to_requeue = {}
       inprogress.each do |unit_of_work|
-        jobs_to_requeue[unit_of_work.queue_name] ||= []
-        jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.message
+        jobs_to_requeue[unit_of_work.queue] ||= []
+        jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
       end
 
-      Sidekiq.redis do |conn|
-        conn.pipelined do
+      redis do |conn|
+        conn.pipelined do |pipeline|
           jobs_to_requeue.each do |queue, jobs|
-            conn.rpush("queue:#{queue}", jobs)
+            pipeline.rpush(queue, jobs)
           end
         end
       end
-      Sidekiq.logger.info("Pushed #{inprogress.size} messages back to Redis")
+      logger.info("Pushed #{inprogress.size} jobs back to Redis")
     rescue => ex
-      Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
-    end
-
-    UnitOfWork = Struct.new(:queue, :message) do
-      def acknowledge
-        # nothing to do
-      end
-
-      def queue_name
-        queue.gsub(/.*queue:/, '')
-      end
-
-      def requeue
-        Sidekiq.redis do |conn|
-          conn.rpush("queue:#{queue_name}", message)
-        end
-      end
+      logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
     end
 
     # Creating the Redis#brpop command takes into account any
@@ -149,8 +76,13 @@ module Sidekiq
     # recreate the queue command each time we invoke Redis#brpop
     # to honor weights and avoid queue starvation.
     def queues_cmd
-      queues = @strictly_ordered_queues ? @unique_queues.dup : @queues.shuffle.uniq
-      queues << Sidekiq::Fetcher::TIMEOUT
+      if @strictly_ordered_queues
+        @queues
+      else
+        permute = @queues.shuffle
+        permute.uniq!
+        permute
+      end
     end
   end
 end

data/lib/sidekiq/iterable_job.rb

@@ -0,0 +1,53 @@
+require "sidekiq/job/iterable"
+
+# Iterable jobs are ones which provide a sequence to process using
+# `build_enumerator(*args, cursor: cursor)` and then process each
+# element of that sequence in `each_iteration(item, *args)`.
+#
+# The job is kicked off as normal:
+#
+#     ProcessUserSet.perform_async(123)
+#
+# but instead of calling `perform`, Sidekiq will call:
+#
+#     enum = ProcessUserSet#build_enumerator(123, cursor:nil)
+#
+# Your Enumerator must yield `(object, updated_cursor)` and
+# Sidekiq will call your `each_iteration` method:
+#
+#     ProcessUserSet#each_iteration(object, 123)
+#
+# After every iteration, Sidekiq will check for shutdown. If we are
+# stopping, the cursor will be saved to Redis and the job re-queued
+# to pick up the rest of the work upon restart. Your job will get
+# the updated_cursor so it can pick up right where it stopped.
+#
+#     enum = ProcessUserSet#build_enumerator(123, cursor: updated_cursor)
+#
+# The cursor object must be serializable to JSON.
+#
+# Note there are several APIs to help you build enumerators for
+# ActiveRecord Relations, CSV files, etc. See sidekiq/job/iterable/*.rb.
+module Sidekiq
+  module IterableJob
+    def self.included(base)
+      base.include Sidekiq::Job
+      base.include Sidekiq::Job::Iterable
+    end
+
+    # def build_enumerator(*args, cursor:)
+    # def each_iteration(item, *args)
+
+    # Your job can also define several callbacks during points
+    # in each job's lifecycle.
+    #
+    # def on_start
+    # def on_resume
+    # def on_stop
+    # def on_complete
+    # def around_iteration
+    #
+    # To keep things simple and compatible, this is the same
+    # API as the `sidekiq-iteration` gem.
+  end
+end

data/lib/sidekiq/job/interrupt_handler.rb

@@ -0,0 +1,22 @@
+module Sidekiq
+  module Job
+    class InterruptHandler
+      include Sidekiq::ServerMiddleware
+
+      def call(instance, hash, queue)
+        yield
+      rescue Interrupted
+        logger.debug "Interrupted, re-queueing..."
+        c = Sidekiq::Client.new
+        c.push(hash)
+        raise Sidekiq::JobRetry::Skip
+      end
+    end
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Job::InterruptHandler
+  end
+end