sidekiq 7.1.6 → 7.3.9

data/lib/sidekiq/api.rb

@@ -4,7 +4,6 @@ require "sidekiq"
 
 require "zlib"
 require "set"
-require "base64"
 
 require "sidekiq/metrics/query"
 
@@ -374,7 +373,7 @@ module Sidekiq
     def display_class
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
       @klass ||= self["display_class"] || begin
-        if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+        if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper" || klass == "Sidekiq::ActiveJob::Wrapper"
           job_class = @item["wrapped"] || args[0]
           if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
             # MailerClass#mailer_method
@@ -390,7 +389,7 @@ module Sidekiq
 
     def display_args
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
-      @display_args ||= if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+      @display_args ||= if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper" || klass == "Sidekiq::ActiveJob::Wrapper"
         job_args = self["wrapped"] ? deserialize_argument(args[0]["arguments"]) : []
         if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
           # remove MailerClass, mailer_method and 'deliver_now'
@@ -491,8 +490,8 @@ module Sidekiq
     end
 
     def uncompress_backtrace(backtrace)
-      decoded = Base64.decode64(backtrace)
-      uncompressed = Zlib::Inflate.inflate(decoded)
+      strict_base64_decoded = backtrace.unpack1("m")
+      uncompressed = Zlib::Inflate.inflate(strict_base64_decoded)
       Sidekiq.load_json(uncompressed)
     end
   end
@@ -669,6 +668,41 @@ module Sidekiq
       end
     end
 
+    def pop_each
+      Sidekiq.redis do |c|
+        size.times do
+          data, score = c.zpopmin(name, 1)&.first
+          break unless data
+          yield data, score
+        end
+      end
+    end
+
+    def retry_all
+      c = Sidekiq::Client.new
+      pop_each do |msg, _|
+        job = Sidekiq.load_json(msg)
+        # Manual retries should not count against the retry limit.
+        job["retry_count"] -= 1 if job["retry_count"]
+        c.push(job)
+      end
+    end
+
+    # Move all jobs from this Set to the Dead Set.
+    # See DeadSet#kill
+    def kill_all(notify_failure: false, ex: nil)
+      ds = DeadSet.new
+      opts = {notify_failure: notify_failure, ex: ex, trim: false}
+
+      begin
+        pop_each do |msg, _|
+          ds.kill(msg, opts)
+        end
+      ensure
+        ds.trim
+      end
+    end
+
     def each
       initial_size = @_size
       offset_size = 0
@@ -679,7 +713,7 @@ module Sidekiq
         range_start = page * page_size + offset_size
         range_end = range_start + page_size - 1
         elements = Sidekiq.redis { |conn|
-          conn.zrange name, range_start, range_end, withscores: true
+          conn.zrange name, range_start, range_end, "withscores"
         }
         break if elements.empty?
         page -= 1
@@ -706,7 +740,7 @@ module Sidekiq
         end
 
       elements = Sidekiq.redis { |conn|
-        conn.zrange(name, begin_score, end_score, "BYSCORE", withscores: true)
+        conn.zrange(name, begin_score, end_score, "BYSCORE", "withscores")
       }
 
       elements.each_with_object([]) do |element, result|
@@ -766,39 +800,21 @@ module Sidekiq
 
   ##
   # The set of scheduled jobs within Sidekiq.
-  # Based on this, you can search/filter for jobs.  Here's an
-  # example where I'm selecting jobs based on some complex logic
-  # and deleting them from the scheduled set.
-  #
   # See the API wiki page for usage notes and examples.
   #
   class ScheduledSet < JobSet
     def initialize
-      super "schedule"
+      super("schedule")
     end
   end
 
   ##
   # The set of retries within Sidekiq.
-  # Based on this, you can search/filter for jobs.  Here's an
-  # example where I'm selecting all jobs of a certain type
-  # and deleting them from the retry queue.
-  #
   # See the API wiki page for usage notes and examples.
   #
   class RetrySet < JobSet
     def initialize
-      super "retry"
-    end
-
-    # Enqueues all jobs pending within the retry set.
-    def retry_all
-      each(&:retry) while size > 0
-    end
-
-    # Kills all jobs pending within the retry set.
-    def kill_all
-      each(&:kill) while size > 0
+      super("retry")
     end
   end
 
@@ -809,36 +825,48 @@ module Sidekiq
   #
   class DeadSet < JobSet
     def initialize
-      super "dead"
+      super("dead")
+    end
+
+    # Trim dead jobs which are over our storage limits
+    def trim
+      hash = Sidekiq.default_configuration
+      now = Time.now.to_f
+      Sidekiq.redis do |conn|
+        conn.multi do |transaction|
+          transaction.zremrangebyscore(name, "-inf", now - hash[:dead_timeout_in_seconds])
+          transaction.zremrangebyrank(name, 0, - hash[:dead_max_jobs])
+        end
+      end
     end
 
     # Add the given job to the Dead set.
     # @param message [String] the job data as JSON
+    # @option opts [Boolean] :notify_failure (true) Whether death handlers should be called
+    # @option opts [Boolean] :trim (true) Whether Sidekiq should trim the structure to keep it within configuration
+    # @option opts [Exception] :ex (RuntimeError) An exception to pass to the death handlers
     def kill(message, opts = {})
       now = Time.now.to_f
       Sidekiq.redis do |conn|
-        conn.multi do |transaction|
-          transaction.zadd(name, now.to_s, message)
-          transaction.zremrangebyscore(name, "-inf", now - Sidekiq::Config::DEFAULTS[:dead_timeout_in_seconds])
-          transaction.zremrangebyrank(name, 0, - Sidekiq::Config::DEFAULTS[:dead_max_jobs])
-        end
+        conn.zadd(name, now.to_s, message)
       end
 
+      trim if opts[:trim] != false
+
       if opts[:notify_failure] != false
         job = Sidekiq.load_json(message)
-        r = RuntimeError.new("Job killed by API")
-        r.set_backtrace(caller)
+        if opts[:ex]
+          ex = opts[:ex]
+        else
+          ex = RuntimeError.new("Job killed by API")
+          ex.set_backtrace(caller)
+        end
         Sidekiq.default_configuration.death_handlers.each do |handle|
-          handle.call(job, r)
+          handle.call(job, ex)
         end
       end
       true
     end
-
-    # Enqueue all dead jobs
-    def retry_all
-      each(&:retry) while size > 0
-    end
   end
 
   ##
@@ -881,7 +909,7 @@ module Sidekiq
     # @api private
     def cleanup
       # dont run cleanup more than once per minute
-      return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
+      return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", "NX", "EX", "60") }
 
       count = 0
       Sidekiq.redis do |conn|
@@ -1110,11 +1138,11 @@ module Sidekiq
 
       procs.zip(all_works).each do |key, workers|
         workers.each_pair do |tid, json|
-          results << [key, tid, Sidekiq.load_json(json)] unless json.empty?
+          results << [key, tid, Sidekiq::Work.new(key, tid, Sidekiq.load_json(json))] unless json.empty?
         end
       end
 
-      results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
+      results.sort_by { |(_, _, hsh)| hsh.raw("run_at") }.each(&block)
     end
 
     # Note that #size is only as accurate as Sidekiq's heartbeat,
@@ -1137,7 +1165,74 @@ module Sidekiq
         end
       end
     end
+
+    ##
+    # Find the work which represents a job with the given JID.
+    # *This is a slow O(n) operation*.  Do not use for app logic.
+    #
+    # @param jid [String] the job identifier
+    # @return [Sidekiq::Work] the work or nil
+    def find_work_by_jid(jid)
+      each do |_process_id, _thread_id, work|
+        job = work.job
+        return work if job.jid == jid
+      end
+      nil
+    end
+  end
+
+  # Sidekiq::Work represents a job which is currently executing.
+  class Work
+    attr_reader :process_id
+    attr_reader :thread_id
+
+    def initialize(pid, tid, hsh)
+      @process_id = pid
+      @thread_id = tid
+      @hsh = hsh
+      @job = nil
+    end
+
+    def queue
+      @hsh["queue"]
+    end
+
+    def run_at
+      Time.at(@hsh["run_at"])
+    end
+
+    def job
+      @job ||= Sidekiq::JobRecord.new(@hsh["payload"])
+    end
+
+    def payload
+      @hsh["payload"]
+    end
+
+    # deprecated
+    def [](key)
+      kwargs = {uplevel: 1}
+      kwargs[:category] = :deprecated if RUBY_VERSION > "3.0" # TODO
+      warn("Direct access to `Sidekiq::Work` attributes is deprecated, please use `#payload`, `#queue`, `#run_at` or `#job` instead", **kwargs)
+
+      @hsh[key]
+    end
+
+    # :nodoc:
+    # @api private
+    def raw(name)
+      @hsh[name]
+    end
+
+    def method_missing(*all)
+      @hsh.send(*all)
+    end
+
+    def respond_to_missing?(name, *args)
+      @hsh.respond_to?(name)
+    end
   end
+
   # Since "worker" is a nebulous term, we've deprecated the use of this class name.
   # Is "worker" a process, a type of job, a thread? Undefined!
   # WorkSet better describes the data.

data/lib/sidekiq/capsule.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "sidekiq/component"
 
 module Sidekiq
@@ -17,6 +19,7 @@ module Sidekiq
   # end
   class Capsule
     include Sidekiq::Component
+    extend Forwardable
 
     attr_reader :name
     attr_reader :queues
@@ -24,6 +27,8 @@ module Sidekiq
     attr_reader :mode
     attr_reader :weights
 
+    def_delegators :@config, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
+
     def initialize(name, config)
       @name = name
       @config = config
@@ -35,9 +40,9 @@ module Sidekiq
 
     def fetcher
       @fetcher ||= begin
-        inst = (config[:fetch_class] || Sidekiq::BasicFetch).new(self)
-        inst.setup(config[:fetch_setup]) if inst.respond_to?(:setup)
-        inst
+        instance = (config[:fetch_class] || Sidekiq::BasicFetch).new(self)
+        instance.setup(config[:fetch_setup]) if instance.respond_to?(:setup)
+        instance
       end
     end
 

data/lib/sidekiq/cli.rb

@@ -38,7 +38,7 @@ module Sidekiq # :nodoc:
     # Code within this method is not tested because it alters
     # global process state irreversibly.  PRs which improve the
     # test coverage of Sidekiq::CLI are welcomed.
-    def run(boot_app: true)
+    def run(boot_app: true, warmup: true)
       boot_application if boot_app
 
       if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
@@ -101,6 +101,8 @@ module Sidekiq # :nodoc:
       # Touch middleware so it isn't lazy loaded by multiple threads, #3043
       @config.server_middleware
 
+      ::Process.warmup if warmup && ::Process.respond_to?(:warmup) && ENV["RUBY_DISABLE_WARMUP"] != "1"
+
       # Before this point, the process is initializing with just the main thread.
       # Starting here the process will now have multiple threads running.
       fire_event(:startup, reverse: false, reraise: true)
@@ -421,3 +423,4 @@ end
 
 require "sidekiq/systemd"
 require "sidekiq/metrics/tracking"
+require "sidekiq/job/interrupt_handler"

data/lib/sidekiq/client.rb

@@ -58,6 +58,23 @@ module Sidekiq
       end
     end
 
+    # Cancel the IterableJob with the given JID.
+    # **NB: Cancellation is asynchronous.** Iteration checks every
+    # five seconds so this will not immediately stop the given job.
+    def cancel!(jid)
+      key = "it-#{jid}"
+      _, result, _ = Sidekiq.redis do |c|
+        c.pipelined do |p|
+          p.hsetnx(key, "cancelled", Time.now.to_i)
+          p.hget(key, "cancelled")
+          p.expire(key, Sidekiq::Job::Iterable::STATE_TTL)
+          # TODO When Redis 7.2 is required
+          # p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
+        end
+      end
+      result.to_i
+    end
+
     ##
     # The main method used to push a job to Redis.  Accepts a number of options:
     #
@@ -248,9 +265,12 @@ module Sidekiq
     def atomic_push(conn, payloads)
       if payloads.first.key?("at")
         conn.zadd("schedule", payloads.flat_map { |hash|
-          at = hash.delete("at").to_s
+          at = hash["at"].to_s
           # ActiveJob sets this but the job has not been enqueued yet
           hash.delete("enqueued_at")
+          # TODO: Use hash.except("at") when support for Ruby 2.7 is dropped
+          hash = hash.dup
+          hash.delete("at")
           [at, Sidekiq.dump_json(hash)]
         })
       else

data/lib/sidekiq/component.rb

@@ -64,5 +64,27 @@ module Sidekiq
       end
       arr.clear if oneshot # once we've fired an event, we never fire it again
     end
+
+    # When you have a large tree of components, the `inspect` output
+    # can get out of hand, especially with lots of Sidekiq::Config
+    # references everywhere. We avoid calling `inspect` on more complex
+    # state and use `to_s` instead to keep output manageable, #6553
+    def inspect
+      "#<#{self.class.name} #{
+        instance_variables.map do |name|
+          value = instance_variable_get(name)
+          case value
+          when Proc
+            "#{name}=#{value}"
+          when Sidekiq::Config
+            "#{name}=#{value}"
+          when Sidekiq::Component
+            "#{name}=#{value}"
+          else
+            "#{name}=#{value.inspect}"
+          end
+        end.join(", ")
+      }>"
+    end
   end
 end

data/lib/sidekiq/config.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "forwardable"
 
 require "set"
@@ -17,6 +19,10 @@ module Sidekiq
       poll_interval_average: nil,
       average_scheduled_poll_interval: 5,
       on_complex_arguments: :raise,
+      iteration: {
+        max_job_runtime: nil,
+        retry_backoff: 0
+      },
       error_handlers: [],
       death_handlers: [],
       lifecycle_events: {
@@ -52,9 +58,15 @@ module Sidekiq
       @capsules = {}
     end
 
-    def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!
+    def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
     attr_reader :capsules
 
+    def inspect
+      "#<#{self.class.name} @options=#{
+        @options.except(:lifecycle_events, :reloader, :death_handlers, :error_handlers).inspect
+      }>"
+    end
+
     def to_json(*)
       Sidekiq.dump_json(@options)
     end
@@ -179,7 +191,13 @@ module Sidekiq
 
     # register global singletons which can be accessed elsewhere
     def register(name, instance)
-      @directory[name] = instance
+      # logger.debug("register[#{name}] = #{instance}")
+      # Sidekiq Enterprise lazy registers a few services so we
+      # can't lock down this hash completely.
+      hash = @directory.dup
+      hash[name] = instance
+      @directory = hash.freeze
+      instance
     end
 
     # find a singleton
@@ -187,10 +205,16 @@ module Sidekiq
       # JNDI is just a fancy name for a hash lookup
       @directory.fetch(name) do |key|
         return nil unless default_class
-        @directory[key] = default_class.new(self)
+        register(key, default_class.new(self))
       end
     end
 
+    def freeze!
+      @directory.freeze
+      @options.freeze
+      true
+    end
+
     ##
     # Death handlers are called when all retries for a job have been exhausted and
     # the job dies.  It's the notification to your application
@@ -258,9 +282,9 @@ module Sidekiq
       @logger = logger
     end
 
-    private def arity(handler)
-      return handler.arity if handler.is_a?(Proc)
-      handler.method(:call).arity
+    private def parameter_size(handler)
+      target = handler.is_a?(Proc) ? handler : handler.method(:call)
+      target.parameters.size
     end
 
     # INTERNAL USE ONLY
@@ -269,7 +293,7 @@ module Sidekiq
         p ["!!!!!", ex]
       end
       @options[:error_handlers].each do |handler|
-        if arity(handler) == 2
+        if parameter_size(handler) == 2
           # TODO Remove in 8.0
           logger.info { "DEPRECATION: Sidekiq exception handlers now take three arguments, see #{handler}" }
           handler.call(ex, {_config: self}.merge(ctx))

data/lib/sidekiq/deploy.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "sidekiq/redis_connection"
 require "time"
 
@@ -34,7 +36,7 @@ module Sidekiq
       # handle an very common error in marking deploys:
       # having every process mark its deploy, leading
       # to N marks for each deploy. Instead we round the time
-      # to the minute so that multple marks within that minute
+      # to the minute so that multiple marks within that minute
       # will all naturally rollup into one mark per minute.
       whence = at.utc
       floor = Time.utc(whence.year, whence.month, whence.mday, whence.hour, whence.min, 0)
@@ -44,7 +46,7 @@ module Sidekiq
 
       @pool.with do |c|
         # only allow one deploy mark for a given label for the next minute
-        lock = c.set("deploylock-#{label}", stamp, nx: true, ex: 60)
+        lock = c.set("deploylock-#{label}", stamp, "nx", "ex", "60")
         if lock
           c.multi do |pipe|
             pipe.hsetnx(key, stamp, label)

data/lib/sidekiq/embedded.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "sidekiq/component"
 require "sidekiq/launcher"
 require "sidekiq/metrics/tracking"

data/lib/sidekiq/fetch.rb

@@ -44,7 +44,7 @@ module Sidekiq # :nodoc:
         return nil
       end
 
-      queue, job = redis { |conn| conn.blocking_call(conn.read_timeout + TIMEOUT, "brpop", *qs, TIMEOUT) }
+      queue, job = redis { |conn| conn.blocking_call(TIMEOUT, "brpop", *qs, TIMEOUT) }
       UnitOfWork.new(queue, job, config) if queue
     end
 

data/lib/sidekiq/iterable_job.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "sidekiq/job/iterable"
+
+# Iterable jobs are ones which provide a sequence to process using
+# `build_enumerator(*args, cursor: cursor)` and then process each
+# element of that sequence in `each_iteration(item, *args)`.
+#
+# The job is kicked off as normal:
+#
+#     ProcessUserSet.perform_async(123)
+#
+# but instead of calling `perform`, Sidekiq will call:
+#
+#     enum = ProcessUserSet#build_enumerator(123, cursor:nil)
+#
+# Your Enumerator must yield `(object, updated_cursor)` and
+# Sidekiq will call your `each_iteration` method:
+#
+#     ProcessUserSet#each_iteration(object, 123)
+#
+# After every iteration, Sidekiq will check for shutdown. If we are
+# stopping, the cursor will be saved to Redis and the job re-queued
+# to pick up the rest of the work upon restart. Your job will get
+# the updated_cursor so it can pick up right where it stopped.
+#
+#     enum = ProcessUserSet#build_enumerator(123, cursor: updated_cursor)
+#
+# The cursor object must be serializable to JSON.
+#
+# Note there are several APIs to help you build enumerators for
+# ActiveRecord Relations, CSV files, etc. See sidekiq/job/iterable/*.rb.
+module Sidekiq
+  module IterableJob
+    def self.included(base)
+      base.include Sidekiq::Job
+      base.include Sidekiq::Job::Iterable
+    end
+
+    # def build_enumerator(*args, cursor:)
+    # def each_iteration(item, *args)
+
+    # Your job can also define several callbacks during points
+    # in each job's lifecycle.
+    #
+    # def on_start
+    # def on_resume
+    # def on_stop
+    # def on_complete
+    # def around_iteration
+    #
+    # To keep things simple and compatible, this is the same
+    # API as the `sidekiq-iteration` gem.
+  end
+end

data/lib/sidekiq/job/interrupt_handler.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Job
+    class InterruptHandler
+      include Sidekiq::ServerMiddleware
+
+      def call(instance, hash, queue)
+        yield
+      rescue Interrupted
+        logger.debug "Interrupted, re-queueing..."
+        c = Sidekiq::Client.new
+        c.push(hash)
+        raise Sidekiq::JobRetry::Skip
+      end
+    end
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Job::InterruptHandler
+  end
+end

data/lib/sidekiq/job/iterable/active_record_enumerator.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Job
+    module Iterable
+      # @api private
+      class ActiveRecordEnumerator
+        def initialize(relation, cursor: nil, **options)
+          @relation = relation
+          @cursor = cursor
+          @options = options
+        end
+
+        def records
+          Enumerator.new(-> { @relation.count }) do |yielder|
+            @relation.find_each(**@options, start: @cursor) do |record|
+              yielder.yield(record, record.id)
+            end
+          end
+        end
+
+        def batches
+          Enumerator.new(-> { @relation.count }) do |yielder|
+            @relation.find_in_batches(**@options, start: @cursor) do |batch|
+              yielder.yield(batch, batch.first.id)
+            end
+          end
+        end
+
+        def relations
+          Enumerator.new(-> { relations_size }) do |yielder|
+            # Convenience to use :batch_size for all the
+            # ActiveRecord batching methods.
+            options = @options.dup
+            options[:of] ||= options.delete(:batch_size)
+
+            @relation.in_batches(**options, start: @cursor) do |relation|
+              first_record = relation.first
+              yielder.yield(relation, first_record.id)
+            end
+          end
+        end
+
+        private
+
+        def relations_size
+          batch_size = @options[:batch_size] || 1000
+          (@relation.count + batch_size - 1) / batch_size # ceiling division
+        end
+      end
+    end
+  end
+end