sidekiq 7.1.4 → 8.0.9

data/lib/sidekiq/client.rb

@@ -58,6 +58,21 @@ module Sidekiq
       end
     end
 
+    # Cancel the IterableJob with the given JID.
+    # **NB: Cancellation is asynchronous.** Iteration checks every
+    # five seconds so this will not immediately stop the given job.
+    def cancel!(jid)
+      key = "it-#{jid}"
+      _, result, _ = Sidekiq.redis do |c|
+        c.pipelined do |p|
+          p.hsetnx(key, "cancelled", Time.now.to_i)
+          p.hget(key, "cancelled")
+          p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
+        end
+      end
+      result.to_i
+    end
+
     ##
     # The main method used to push a job to Redis.  Accepts a number of options:
     #
@@ -74,7 +89,7 @@ module Sidekiq
     #
     # Any options valid for a job class's sidekiq_options are also available here.
     #
-    # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
+    # All keys must be strings, not symbols.  NB: because we are serializing to JSON, all
     # symbols in 'args' will be converted to strings.  Note that +backtrace: true+ can take quite a bit of
     # space in Redis; a large volume of failing jobs can start Redis swapping if you aren't careful.
     #
@@ -102,6 +117,9 @@ module Sidekiq
     # larger than 1000 but YMMV based on network quality, size of job args, etc.
     # A large number of jobs can cause a bit of Redis command processing latency.
     #
+    # Accepts an additional `:spread_interval` option (in seconds) to randomly spread
+    # the jobs schedule times over the specified interval.
+    #
     # Takes the same arguments as #push except that args is expected to be
     # an Array of Arrays.  All other keys are duplicated for each job.  Each job
     # is run through the client middleware pipeline and each job gets its own Job ID
@@ -111,32 +129,45 @@ module Sidekiq
     # prevented a job push.
     #
     # Example (pushing jobs in batches):
-    #   push_bulk('class' => 'MyJob', 'args' => (1..100_000).to_a, batch_size: 1_000)
+    #   push_bulk('class' => MyJob, 'args' => (1..100_000).to_a, batch_size: 1_000)
     #
     def push_bulk(items)
       batch_size = items.delete(:batch_size) || items.delete("batch_size") || 1_000
       args = items["args"]
-      at = items.delete("at")
+      at = items.delete("at") || items.delete(:at)
       raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
       raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
       jid = items.delete("jid")
       raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
 
+      spread_interval = items.delete(:spread_interval) || items.delete("spread_interval")
+      raise ArgumentError, "Jobs 'spread_interval' must be a positive Numeric" if spread_interval && (!spread_interval.is_a?(Numeric) || spread_interval <= 0)
+      raise ArgumentError, "Only one of 'at' or 'spread_interval' can be provided" if at && spread_interval
+
+      if !at && spread_interval
+        # Do not use spread interval smaller than pooling interval.
+        spread_interval = [spread_interval, 5].max
+        now = Time.now.to_f
+        at = args.map { now + rand * spread_interval }
+      end
+
       normed = normalize_item(items)
+      slice_index = 0
       result = args.each_slice(batch_size).flat_map do |slice|
         raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless slice.is_a?(Array) && slice.all?(Array)
         break [] if slice.empty? # no jobs to push
 
         payloads = slice.map.with_index { |job_args, index|
           copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
-          copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
+          copy["at"] = (at.is_a?(Array) ? at[slice_index + index] : at) if at
           result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
             verify_json(copy)
             copy
           end
           result || nil
         }
+        slice_index += batch_size
 
         to_push = payloads.compact
         raw_push(to_push) unless to_push.empty?
@@ -246,20 +277,22 @@ module Sidekiq
     def atomic_push(conn, payloads)
       if payloads.first.key?("at")
         conn.zadd("schedule", payloads.flat_map { |hash|
-          at = hash.delete("at").to_s
-          # ActiveJob sets this but the job has not been enqueued yet
-          hash.delete("enqueued_at")
+          at = hash["at"].to_s
+          # ActiveJob sets enqueued_at but the job has not been enqueued yet
+          hash = hash.except("enqueued_at", "at")
           [at, Sidekiq.dump_json(hash)]
         })
       else
-        queue = payloads.first["queue"]
-        now = Time.now.to_f
-        to_push = payloads.map { |entry|
-          entry["enqueued_at"] = now
-          Sidekiq.dump_json(entry)
-        }
-        conn.sadd("queues", [queue])
-        conn.lpush("queue:#{queue}", to_push)
+        now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) # milliseconds since the epoch
+        grouped_queues = payloads.group_by { |job| job["queue"] }
+        conn.sadd("queues", grouped_queues.keys)
+        grouped_queues.each do |queue, grouped_payloads|
+          to_push = grouped_payloads.map { |entry|
+            entry["enqueued_at"] = now
+            Sidekiq.dump_json(entry)
+          }
+          conn.lpush("queue:#{queue}", to_push)
+        end
       end
     end
   end

data/lib/sidekiq/component.rb

@@ -1,11 +1,39 @@
 # frozen_string_literal: true
 
 module Sidekiq
+  # Ruby's default thread priority is 0, which uses 100ms time slices.
+  # This can lead to some surprising thread starvation; if using a lot of
+  # CPU-heavy concurrency, it may take several seconds before a Thread gets
+  # on the CPU.
+  #
+  # Negative priorities lower the timeslice by half, so -1 = 50ms, -2 = 25ms, etc.
+  # With more frequent timeslices, we reduce the risk of unintentional timeouts
+  # and starvation.
+  #
+  # Customize like so:
+  #
+  #   Sidekiq.configure_server do |cfg|
+  #     cfg.thread_priority = 0
+  #   end
+  #
+  DEFAULT_THREAD_PRIORITY = -1
+
   ##
-  # Sidekiq::Component assumes a config instance is available at @config
+  # Sidekiq::Component provides a set of utility methods depending only
+  # on Sidekiq::Config. It assumes a config instance is available at @config.
   module Component # :nodoc:
     attr_reader :config
 
+    # This is epoch milliseconds, appropriate for persistence
+    def real_ms
+      ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
+    end
+
+    # used for time difference and relative comparisons, not persistence.
+    def mono_ms
+      ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
+    end
+
     def watchdog(last_words)
       yield
     rescue Exception => ex
@@ -13,11 +41,11 @@ module Sidekiq
       raise ex
     end
 
-    def safe_thread(name, &block)
+    def safe_thread(name, priority: nil, &block)
       Thread.new do
         Thread.current.name = "sidekiq.#{name}"
         watchdog(name, &block)
-      end
+      end.tap { |t| t.priority = (priority || config.thread_priority || DEFAULT_THREAD_PRIORITY) }
     end
 
     def logger
@@ -64,5 +92,38 @@ module Sidekiq
       end
       arr.clear if oneshot # once we've fired an event, we never fire it again
     end
+
+    # When you have a large tree of components, the `inspect` output
+    # can get out of hand, especially with lots of Sidekiq::Config
+    # references everywhere. We avoid calling `inspect` on more complex
+    # state and use `to_s` instead to keep output manageable, #6553
+    def inspect
+      "#<#{self.class.name} #{
+        instance_variables.map do |name|
+          value = instance_variable_get(name)
+          case value
+          when Proc
+            "#{name}=#{value}"
+          when Sidekiq::Config
+            "#{name}=#{value}"
+          when Sidekiq::Component
+            "#{name}=#{value}"
+          else
+            "#{name}=#{value.inspect}"
+          end
+        end.join(", ")
+      }>"
+    end
+
+    def default_tag(dir = Dir.pwd)
+      name = File.basename(dir)
+      prevdir = File.dirname(dir) # Capistrano release directory?
+      if name.to_i != 0 && prevdir
+        if File.basename(prevdir) == "releases"
+          return File.basename(File.dirname(prevdir))
+        end
+      end
+      name
+    end
   end
 end

data/lib/sidekiq/config.rb

@@ -1,6 +1,6 @@
-require "forwardable"
+# frozen_string_literal: true
 
-require "set"
+require "forwardable"
 require "sidekiq/redis_connection"
 
 module Sidekiq
@@ -17,12 +17,16 @@ module Sidekiq
       poll_interval_average: nil,
       average_scheduled_poll_interval: 5,
       on_complex_arguments: :raise,
+      # if the Iterable job runs longer than this value (in seconds), then the job
+      # will be interrupted after the current iteration and re-enqueued at the back of the queue
+      max_iteration_runtime: nil,
       error_handlers: [],
       death_handlers: [],
       lifecycle_events: {
         startup: [],
         quiet: [],
         shutdown: [],
+        exit: [],
         # triggers when we fire the first heartbeat on startup OR repairing a network partition
         heartbeat: [],
         # triggers on EVERY heartbeat call, every 10 seconds
@@ -31,17 +35,27 @@ module Sidekiq
       dead_max_jobs: 10_000,
       dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
       reloader: proc { |&block| block.call },
-      backtrace_cleaner: ->(backtrace) { backtrace }
+      backtrace_cleaner: ->(backtrace) { backtrace },
+      logged_job_attributes: ["bid", "tags"]
     }
 
-    ERROR_HANDLER = ->(ex, ctx) {
-      cfg = ctx[:_config] || Sidekiq.default_configuration
-      l = cfg.logger
-      l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
-      l.warn("#{ex.class.name}: #{ex.message}")
-      unless ex.backtrace.nil?
-        backtrace = cfg[:backtrace_cleaner].call(ex.backtrace)
-        l.warn(backtrace.join("\n"))
+    ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
+      Sidekiq::Context.with(ctx) do
+        dev = cfg[:environment] == "development"
+        fancy = dev && $stdout.tty? # 🎩
+        # Weird logic here but we want to show the backtrace in local
+        # development or if verbose logging is enabled.
+        #
+        # `full_message` contains the error class, message and backtrace
+        # `detailed_message` contains the error class and message
+        #
+        # Absolutely terrible API names. Not useful at all to have two
+        # methods with similar but obscure names.
+        if dev || cfg.logger.debug?
+          cfg.logger.info { ex.full_message(highlight: fancy) }
+        else
+          cfg.logger.info { ex.detailed_message(highlight: fancy) }
+        end
       end
     }
 
@@ -53,8 +67,15 @@ module Sidekiq
       @capsules = {}
     end
 
-    def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!
+    def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
     attr_reader :capsules
+    attr_accessor :thread_priority
+
+    def inspect
+      "#<#{self.class.name} @options=#{
+        @options.except(:lifecycle_events, :reloader, :death_handlers, :error_handlers).inspect
+      }>"
+    end
 
     def to_json(*)
       Sidekiq.dump_json(@options)
@@ -124,11 +145,16 @@ module Sidekiq
       @redis_config = @redis_config.merge(hash)
     end
 
+    def reap_idle_redis_connections(timeout = nil)
+      self[:reap_connections] = timeout
+    end
+    alias_method :reap, :reap_idle_redis_connections
+
     def redis_pool
       Thread.current[:sidekiq_redis_pool] || Thread.current[:sidekiq_capsule]&.redis_pool || local_redis_pool
     end
 
-    private def local_redis_pool
+    def local_redis_pool
       # this is our internal client/housekeeping pool. each capsule has its
       # own pool for executing threads.
       @redis ||= new_redis_pool(10, "internal")
@@ -180,7 +206,13 @@ module Sidekiq
 
     # register global singletons which can be accessed elsewhere
     def register(name, instance)
-      @directory[name] = instance
+      # logger.debug("register[#{name}] = #{instance}")
+      # Sidekiq Enterprise lazy registers a few services so we
+      # can't lock down this hash completely.
+      hash = @directory.dup
+      hash[name] = instance
+      @directory = hash.freeze
+      instance
     end
 
     # find a singleton
@@ -188,10 +220,16 @@ module Sidekiq
       # JNDI is just a fancy name for a hash lookup
       @directory.fetch(name) do |key|
         return nil unless default_class
-        @directory[key] = default_class.new(self)
+        register(key, default_class.new(self))
       end
     end
 
+    def freeze!
+      @directory.freeze
+      @options.freeze
+      true
+    end
+
     ##
     # Death handlers are called when all retries for a job have been exhausted and
     # the job dies.  It's the notification to your application
@@ -226,7 +264,7 @@ module Sidekiq
     end
 
     # Register a block to run at a point in the Sidekiq lifecycle.
-    # :startup, :quiet or :shutdown are valid events.
+    # :startup, :quiet, :shutdown, or :exit are valid events.
     #
     #   Sidekiq.configure_server do |config|
     #     config.on(:shutdown) do
@@ -259,14 +297,18 @@ module Sidekiq
       @logger = logger
     end
 
+    private def parameter_size(handler)
+      target = handler.is_a?(Proc) ? handler : handler.method(:call)
+      target.parameters.size
+    end
+
     # INTERNAL USE ONLY
     def handle_exception(ex, ctx = {})
       if @options[:error_handlers].size == 0
         p ["!!!!!", ex]
       end
-      ctx[:_config] = self
       @options[:error_handlers].each do |handler|
-        handler.call(ex, ctx)
+        handler.call(ex, ctx, self)
       rescue Exception => e
         l = logger
         l.error "!!! ERROR HANDLER THREW AN ERROR !!!"

data/lib/sidekiq/deploy.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "sidekiq/redis_connection"
 require "time"
 
@@ -34,7 +36,7 @@ module Sidekiq
       # handle an very common error in marking deploys:
       # having every process mark its deploy, leading
       # to N marks for each deploy. Instead we round the time
-      # to the minute so that multple marks within that minute
+      # to the minute so that multiple marks within that minute
       # will all naturally rollup into one mark per minute.
       whence = at.utc
       floor = Time.utc(whence.year, whence.month, whence.mday, whence.hour, whence.min, 0)
@@ -44,7 +46,7 @@ module Sidekiq
 
       @pool.with do |c|
         # only allow one deploy mark for a given label for the next minute
-        lock = c.set("deploylock-#{label}", stamp, nx: true, ex: 60)
+        lock = c.set("deploylock-#{label}", stamp, "nx", "ex", "60")
         if lock
           c.multi do |pipe|
             pipe.hsetnx(key, stamp, label)

data/lib/sidekiq/embedded.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "sidekiq/component"
 require "sidekiq/launcher"
 require "sidekiq/metrics/tracking"
@@ -32,6 +34,7 @@ module Sidekiq
     private
 
     def housekeeping
+      @config[:tag] ||= default_tag
       logger.info "Running in #{RUBY_DESCRIPTION}"
       logger.info Sidekiq::LICENSE
       logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
@@ -40,7 +43,7 @@ module Sidekiq
       # fire startup and start multithreading.
       info = config.redis_info
       ver = Gem::Version.new(info["redis_version"])
-      raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
+      raise "You are connected to Redis #{ver}, Sidekiq requires Redis 7.0.0 or greater" if ver < Gem::Version.new("7.0.0")
 
       maxmemory_policy = info["maxmemory_policy"]
       if maxmemory_policy != "noeviction"

data/lib/sidekiq/fetch.rb

@@ -7,6 +7,7 @@ require "sidekiq/capsule"
 module Sidekiq # :nodoc:
   class BasicFetch
     include Sidekiq::Component
+
     # We want the fetch operation to timeout every few seconds so the thread
     # can check if the process is shutting down.
     TIMEOUT = 2
@@ -44,7 +45,7 @@ module Sidekiq # :nodoc:
         return nil
       end
 
-      queue, job = redis { |conn| conn.blocking_call(conn.read_timeout + TIMEOUT, "brpop", *qs, TIMEOUT) }
+      queue, job = redis { |conn| conn.blocking_call(TIMEOUT, "brpop", *qs, TIMEOUT) }
       UnitOfWork.new(queue, job, config) if queue
     end
 

data/lib/sidekiq/iterable_job.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "sidekiq/job/iterable"
+
+# Iterable jobs are ones which provide a sequence to process using
+# `build_enumerator(*args, cursor: cursor)` and then process each
+# element of that sequence in `each_iteration(item, *args)`.
+#
+# The job is kicked off as normal:
+#
+#     ProcessUserSet.perform_async(123)
+#
+# but instead of calling `perform`, Sidekiq will call:
+#
+#     enum = ProcessUserSet#build_enumerator(123, cursor:nil)
+#
+# Your Enumerator must yield `(object, updated_cursor)` and
+# Sidekiq will call your `each_iteration` method:
+#
+#     ProcessUserSet#each_iteration(object, 123)
+#
+# After every iteration, Sidekiq will check for shutdown. If we are
+# stopping, the cursor will be saved to Redis and the job re-queued
+# to pick up the rest of the work upon restart. Your job will get
+# the updated_cursor so it can pick up right where it stopped.
+#
+#     enum = ProcessUserSet#build_enumerator(123, cursor: updated_cursor)
+#
+# The cursor object must be serializable to JSON.
+#
+# Note there are several APIs to help you build enumerators for
+# ActiveRecord Relations, CSV files, etc. See sidekiq/job/iterable/*.rb.
+module Sidekiq
+  module IterableJob
+    def self.included(base)
+      base.include Sidekiq::Job
+      base.include Sidekiq::Job::Iterable
+    end
+
+    # def build_enumerator(*args, cursor:)
+    # def each_iteration(item, *args)
+
+    # Your job can also define several callbacks during points
+    # in each job's lifecycle.
+    #
+    # def on_start
+    # def on_resume
+    # def on_stop
+    # def on_cancel
+    # def on_complete
+    # def around_iteration
+    #
+    # To keep things simple and compatible, this is the same
+    # API as the `sidekiq-iteration` gem.
+  end
+end

data/lib/sidekiq/job/interrupt_handler.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Job
+    class InterruptHandler
+      include Sidekiq::ServerMiddleware
+
+      def call(instance, hash, queue)
+        yield
+      rescue Interrupted
+        logger.debug "Interrupted, re-queueing..."
+        c = Sidekiq::Client.new
+        c.push(hash)
+        raise Sidekiq::JobRetry::Skip
+      end
+    end
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Job::InterruptHandler
+  end
+end

data/lib/sidekiq/job/iterable/active_record_enumerator.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Job
+    module Iterable
+      # @api private
+      class ActiveRecordEnumerator
+        def initialize(relation, cursor: nil, **options)
+          @relation = relation
+          @cursor = cursor
+          @options = options
+        end
+
+        def records
+          Enumerator.new(-> { @relation.count }) do |yielder|
+            @relation.find_each(**@options, start: @cursor) do |record|
+              yielder.yield(record, record.id)
+            end
+          end
+        end
+
+        def batches
+          Enumerator.new(-> { @relation.count }) do |yielder|
+            @relation.find_in_batches(**@options, start: @cursor) do |batch|
+              yielder.yield(batch, batch.first.id)
+            end
+          end
+        end
+
+        def relations
+          Enumerator.new(-> { relations_size }) do |yielder|
+            # Convenience to use :batch_size for all the
+            # ActiveRecord batching methods.
+            options = @options.dup
+            options[:of] ||= options.delete(:batch_size)
+
+            @relation.in_batches(**options, start: @cursor) do |relation|
+              first_record = relation.first
+              yielder.yield(relation, first_record.id)
+            end
+          end
+        end
+
+        private
+
+        def relations_size
+          batch_size = @options[:batch_size] || 1000
+          (@relation.count + batch_size - 1) / batch_size # ceiling division
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/job/iterable/csv_enumerator.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Job
+    module Iterable
+      # @api private
+      class CsvEnumerator
+        def initialize(csv)
+          unless defined?(CSV) && csv.instance_of?(CSV)
+            raise ArgumentError, "CsvEnumerator.new takes CSV object"
+          end
+
+          @csv = csv
+        end
+
+        def rows(cursor:)
+          @csv.lazy
+            .each_with_index
+            .drop(cursor || 0)
+            .to_enum { count_of_rows_in_file }
+        end
+
+        def batches(cursor:, batch_size: 100)
+          @csv.lazy
+            .each_slice(batch_size)
+            .with_index
+            .drop(cursor || 0)
+            .to_enum { (count_of_rows_in_file.to_f / batch_size).ceil }
+        end
+
+        private
+
+        def count_of_rows_in_file
+          filepath = @csv.path
+          return unless filepath
+
+          count = IO.popen(["wc", "-l", filepath]) do |out|
+            out.read.strip.to_i
+          end
+
+          count -= 1 if @csv.headers
+          count
+        end
+      end
+    end
+  end
+end