sidekiq 7.0.9 → 7.3.10

data/lib/active_job/queue_adapters/sidekiq_adapter.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module QueueAdapters
+    # Explicitly remove the implementation existing in older rails'.
+    remove_const(:SidekiqAdapter) if const_defined?(:SidekiqAdapter)
+
+    # Sidekiq adapter for Active Job
+    #
+    # To use Sidekiq set the queue_adapter config to +:sidekiq+.
+    #
+    #   Rails.application.config.active_job.queue_adapter = :sidekiq
+    class SidekiqAdapter
+      # Defines whether enqueuing should happen implicitly to after commit when called
+      # from inside a transaction.
+      # @api private
+      def enqueue_after_transaction_commit?
+        true
+      end
+
+      # @api private
+      def enqueue(job)
+        job.provider_job_id = JobWrapper.set(
+          wrapped: job.class,
+          queue: job.queue_name
+        ).perform_async(job.serialize)
+      end
+
+      # @api private
+      def enqueue_at(job, timestamp)
+        job.provider_job_id = JobWrapper.set(
+          wrapped: job.class,
+          queue: job.queue_name
+        ).perform_at(timestamp, job.serialize)
+      end
+
+      # @api private
+      def enqueue_all(jobs)
+        enqueued_count = 0
+        jobs.group_by(&:class).each do |job_class, same_class_jobs|
+          same_class_jobs.group_by(&:queue_name).each do |queue, same_class_and_queue_jobs|
+            immediate_jobs, scheduled_jobs = same_class_and_queue_jobs.partition { |job| job.scheduled_at.nil? }
+
+            if immediate_jobs.any?
+              jids = Sidekiq::Client.push_bulk(
+                "class" => JobWrapper,
+                "wrapped" => job_class,
+                "queue" => queue,
+                "args" => immediate_jobs.map { |job| [job.serialize] }
+              )
+              enqueued_count += jids.compact.size
+            end
+
+            if scheduled_jobs.any?
+              jids = Sidekiq::Client.push_bulk(
+                "class" => JobWrapper,
+                "wrapped" => job_class,
+                "queue" => queue,
+                "args" => scheduled_jobs.map { |job| [job.serialize] },
+                "at" => scheduled_jobs.map { |job| job.scheduled_at&.to_f }
+              )
+              enqueued_count += jids.compact.size
+            end
+          end
+        end
+        enqueued_count
+      end
+
+      # Defines a class alias for backwards compatibility with enqueued Active Job jobs.
+      # @api private
+      class JobWrapper < Sidekiq::ActiveJob::Wrapper
+      end
+    end
+  end
+end

data/lib/generators/sidekiq/job_generator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "rails/generators/named_base"
 
 module Sidekiq

data/lib/sidekiq/api.rb

@@ -4,7 +4,6 @@ require "sidekiq"
 
 require "zlib"
 require "set"
-require "base64"
 
 require "sidekiq/metrics/query"
 
@@ -92,11 +91,11 @@ module Sidekiq
           pipeline.zcard("retry")
           pipeline.zcard("dead")
           pipeline.scard("processes")
-          pipeline.lrange("queue:default", -1, -1)
+          pipeline.lindex("queue:default", -1)
         end
       }
 
-      default_queue_latency = if (entry = pipe1_res[6].first)
+      default_queue_latency = if (entry = pipe1_res[6])
         job = begin
           Sidekiq.load_json(entry)
         rescue
@@ -264,8 +263,8 @@ module Sidekiq
     # @return [Float] in seconds
     def latency
       entry = Sidekiq.redis { |conn|
-        conn.lrange(@rname, -1, -1)
-      }.first
+        conn.lindex(@rname, -1)
+      }
       return 0 unless entry
       job = Sidekiq.load_json(entry)
       now = Time.now.to_f
@@ -374,7 +373,7 @@ module Sidekiq
     def display_class
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
       @klass ||= self["display_class"] || begin
-        if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+        if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper" || klass == "Sidekiq::ActiveJob::Wrapper"
           job_class = @item["wrapped"] || args[0]
           if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
             # MailerClass#mailer_method
@@ -390,14 +389,14 @@ module Sidekiq
 
     def display_args
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
-      @display_args ||= if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
-        job_args = self["wrapped"] ? args[0]["arguments"] : []
+      @display_args ||= if klass == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper" || klass == "Sidekiq::ActiveJob::Wrapper"
+        job_args = self["wrapped"] ? deserialize_argument(args[0]["arguments"]) : []
         if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
           # remove MailerClass, mailer_method and 'deliver_now'
           job_args.drop(3)
         elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
           # remove MailerClass, mailer_method and 'deliver_now'
-          job_args.drop(3).first["args"]
+          job_args.drop(3).first.values_at("params", "args")
         else
           job_args
         end
@@ -467,9 +466,32 @@ module Sidekiq
 
     private
 
+    ACTIVE_JOB_PREFIX = "_aj_"
+    GLOBALID_KEY = "_aj_globalid"
+
+    def deserialize_argument(argument)
+      case argument
+      when Array
+        argument.map { |arg| deserialize_argument(arg) }
+      when Hash
+        if serialized_global_id?(argument)
+          argument[GLOBALID_KEY]
+        else
+          argument.transform_values { |v| deserialize_argument(v) }
+            .reject { |k, _| k.start_with?(ACTIVE_JOB_PREFIX) }
+        end
+      else
+        argument
+      end
+    end
+
+    def serialized_global_id?(hash)
+      hash.size == 1 && hash.include?(GLOBALID_KEY)
+    end
+
     def uncompress_backtrace(backtrace)
-      decoded = Base64.decode64(backtrace)
-      uncompressed = Zlib::Inflate.inflate(decoded)
+      strict_base64_decoded = backtrace.unpack1("m")
+      uncompressed = Zlib::Inflate.inflate(strict_base64_decoded)
       Sidekiq.load_json(uncompressed)
     end
   end
@@ -548,7 +570,7 @@ module Sidekiq
     def remove_job
       Sidekiq.redis do |conn|
         results = conn.multi { |transaction|
-          transaction.zrangebyscore(parent.name, score, score)
+          transaction.zrange(parent.name, score, score, "BYSCORE")
           transaction.zremrangebyscore(parent.name, score, score)
         }.first
 
@@ -646,6 +668,41 @@ module Sidekiq
       end
     end
 
+    def pop_each
+      Sidekiq.redis do |c|
+        size.times do
+          data, score = c.zpopmin(name, 1)&.first
+          break unless data
+          yield data, score
+        end
+      end
+    end
+
+    def retry_all
+      c = Sidekiq::Client.new
+      pop_each do |msg, _|
+        job = Sidekiq.load_json(msg)
+        # Manual retries should not count against the retry limit.
+        job["retry_count"] -= 1 if job["retry_count"]
+        c.push(job)
+      end
+    end
+
+    # Move all jobs from this Set to the Dead Set.
+    # See DeadSet#kill
+    def kill_all(notify_failure: false, ex: nil)
+      ds = DeadSet.new
+      opts = {notify_failure: notify_failure, ex: ex, trim: false}
+
+      begin
+        pop_each do |msg, _|
+          ds.kill(msg, opts)
+        end
+      ensure
+        ds.trim
+      end
+    end
+
     def each
       initial_size = @_size
       offset_size = 0
@@ -656,7 +713,7 @@ module Sidekiq
         range_start = page * page_size + offset_size
         range_end = range_start + page_size - 1
         elements = Sidekiq.redis { |conn|
-          conn.zrange name, range_start, range_end, withscores: true
+          conn.zrange name, range_start, range_end, "withscores"
         }
         break if elements.empty?
         page -= 1
@@ -683,7 +740,7 @@ module Sidekiq
         end
 
       elements = Sidekiq.redis { |conn|
-        conn.zrangebyscore(name, begin_score, end_score, withscores: true)
+        conn.zrange(name, begin_score, end_score, "BYSCORE", "withscores")
       }
 
       elements.each_with_object([]) do |element, result|
@@ -724,7 +781,7 @@ module Sidekiq
     # @api private
     def delete_by_jid(score, jid)
       Sidekiq.redis do |conn|
-        elements = conn.zrangebyscore(name, score, score)
+        elements = conn.zrange(name, score, score, "BYSCORE")
         elements.each do |element|
           if element.index(jid)
             message = Sidekiq.load_json(element)
@@ -743,39 +800,21 @@ module Sidekiq
 
   ##
   # The set of scheduled jobs within Sidekiq.
-  # Based on this, you can search/filter for jobs.  Here's an
-  # example where I'm selecting jobs based on some complex logic
-  # and deleting them from the scheduled set.
-  #
   # See the API wiki page for usage notes and examples.
   #
   class ScheduledSet < JobSet
     def initialize
-      super "schedule"
+      super("schedule")
     end
   end
 
   ##
   # The set of retries within Sidekiq.
-  # Based on this, you can search/filter for jobs.  Here's an
-  # example where I'm selecting all jobs of a certain type
-  # and deleting them from the retry queue.
-  #
   # See the API wiki page for usage notes and examples.
   #
   class RetrySet < JobSet
     def initialize
-      super "retry"
-    end
-
-    # Enqueues all jobs pending within the retry set.
-    def retry_all
-      each(&:retry) while size > 0
-    end
-
-    # Kills all jobs pending within the retry set.
-    def kill_all
-      each(&:kill) while size > 0
+      super("retry")
     end
   end
 
@@ -786,36 +825,48 @@ module Sidekiq
   #
   class DeadSet < JobSet
     def initialize
-      super "dead"
+      super("dead")
+    end
+
+    # Trim dead jobs which are over our storage limits
+    def trim
+      hash = Sidekiq.default_configuration
+      now = Time.now.to_f
+      Sidekiq.redis do |conn|
+        conn.multi do |transaction|
+          transaction.zremrangebyscore(name, "-inf", now - hash[:dead_timeout_in_seconds])
+          transaction.zremrangebyrank(name, 0, - hash[:dead_max_jobs])
+        end
+      end
     end
 
     # Add the given job to the Dead set.
     # @param message [String] the job data as JSON
+    # @option opts [Boolean] :notify_failure (true) Whether death handlers should be called
+    # @option opts [Boolean] :trim (true) Whether Sidekiq should trim the structure to keep it within configuration
+    # @option opts [Exception] :ex (RuntimeError) An exception to pass to the death handlers
     def kill(message, opts = {})
       now = Time.now.to_f
       Sidekiq.redis do |conn|
-        conn.multi do |transaction|
-          transaction.zadd(name, now.to_s, message)
-          transaction.zremrangebyscore(name, "-inf", now - Sidekiq::Config::DEFAULTS[:dead_timeout_in_seconds])
-          transaction.zremrangebyrank(name, 0, - Sidekiq::Config::DEFAULTS[:dead_max_jobs])
-        end
+        conn.zadd(name, now.to_s, message)
       end
 
+      trim if opts[:trim] != false
+
       if opts[:notify_failure] != false
         job = Sidekiq.load_json(message)
-        r = RuntimeError.new("Job killed by API")
-        r.set_backtrace(caller)
+        if opts[:ex]
+          ex = opts[:ex]
+        else
+          ex = RuntimeError.new("Job killed by API")
+          ex.set_backtrace(caller)
+        end
         Sidekiq.default_configuration.death_handlers.each do |handle|
-          handle.call(job, r)
+          handle.call(job, ex)
         end
       end
       true
     end
-
-    # Enqueue all dead jobs
-    def retry_all
-      each(&:retry) while size > 0
-    end
   end
 
   ##
@@ -858,7 +909,7 @@ module Sidekiq
     # @api private
     def cleanup
       # dont run cleanup more than once per minute
-      return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
+      return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", "NX", "EX", "60") }
 
       count = 0
       Sidekiq.redis do |conn|
@@ -1087,11 +1138,11 @@ module Sidekiq
 
       procs.zip(all_works).each do |key, workers|
         workers.each_pair do |tid, json|
-          results << [key, tid, Sidekiq.load_json(json)] unless json.empty?
+          results << [key, tid, Sidekiq::Work.new(key, tid, Sidekiq.load_json(json))] unless json.empty?
         end
       end
 
-      results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
+      results.sort_by { |(_, _, hsh)| hsh.raw("run_at") }.each(&block)
     end
 
     # Note that #size is only as accurate as Sidekiq's heartbeat,
@@ -1114,7 +1165,74 @@ module Sidekiq
         end
       end
     end
+
+    ##
+    # Find the work which represents a job with the given JID.
+    # *This is a slow O(n) operation*.  Do not use for app logic.
+    #
+    # @param jid [String] the job identifier
+    # @return [Sidekiq::Work] the work or nil
+    def find_work_by_jid(jid)
+      each do |_process_id, _thread_id, work|
+        job = work.job
+        return work if job.jid == jid
+      end
+      nil
+    end
   end
+
+  # Sidekiq::Work represents a job which is currently executing.
+  class Work
+    attr_reader :process_id
+    attr_reader :thread_id
+
+    def initialize(pid, tid, hsh)
+      @process_id = pid
+      @thread_id = tid
+      @hsh = hsh
+      @job = nil
+    end
+
+    def queue
+      @hsh["queue"]
+    end
+
+    def run_at
+      Time.at(@hsh["run_at"])
+    end
+
+    def job
+      @job ||= Sidekiq::JobRecord.new(@hsh["payload"])
+    end
+
+    def payload
+      @hsh["payload"]
+    end
+
+    # deprecated
+    def [](key)
+      kwargs = {uplevel: 1}
+      kwargs[:category] = :deprecated if RUBY_VERSION > "3.0" # TODO
+      warn("Direct access to `Sidekiq::Work` attributes is deprecated, please use `#payload`, `#queue`, `#run_at` or `#job` instead", **kwargs)
+
+      @hsh[key]
+    end
+
+    # :nodoc:
+    # @api private
+    def raw(name)
+      @hsh[name]
+    end
+
+    def method_missing(*all)
+      @hsh.send(*all)
+    end
+
+    def respond_to_missing?(name, *args)
+      @hsh.respond_to?(name)
+    end
+  end
+
   # Since "worker" is a nebulous term, we've deprecated the use of this class name.
   # Is "worker" a process, a type of job, a thread? Undefined!
   # WorkSet better describes the data.

data/lib/sidekiq/capsule.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "sidekiq/component"
 
 module Sidekiq
@@ -17,6 +19,7 @@ module Sidekiq
   # end
   class Capsule
     include Sidekiq::Component
+    extend Forwardable
 
     attr_reader :name
     attr_reader :queues
@@ -24,6 +27,8 @@ module Sidekiq
     attr_reader :mode
     attr_reader :weights
 
+    def_delegators :@config, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
+
     def initialize(name, config)
       @name = name
       @config = config
@@ -35,9 +40,9 @@ module Sidekiq
 
     def fetcher
       @fetcher ||= begin
-        inst = (config[:fetch_class] || Sidekiq::BasicFetch).new(self)
-        inst.setup(config[:fetch_setup]) if inst.respond_to?(:setup)
-        inst
+        instance = (config[:fetch_class] || Sidekiq::BasicFetch).new(self)
+        instance.setup(config[:fetch_setup]) if instance.respond_to?(:setup)
+        instance
       end
     end
 

data/lib/sidekiq/cli.rb

@@ -38,7 +38,7 @@ module Sidekiq # :nodoc:
     # Code within this method is not tested because it alters
     # global process state irreversibly.  PRs which improve the
     # test coverage of Sidekiq::CLI are welcomed.
-    def run(boot_app: true)
+    def run(boot_app: true, warmup: true)
       boot_application if boot_app
 
       if environment == "development" && $stdout.tty? && @config.logger.formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
@@ -101,6 +101,8 @@ module Sidekiq # :nodoc:
       # Touch middleware so it isn't lazy loaded by multiple threads, #3043
       @config.server_middleware
 
+      ::Process.warmup if warmup && ::Process.respond_to?(:warmup) && ENV["RUBY_DISABLE_WARMUP"] != "1"
+
       # Before this point, the process is initializing with just the main thread.
       # Starting here the process will now have multiple threads running.
       fire_event(:startup, reverse: false, reraise: true)
@@ -230,6 +232,7 @@ module Sidekiq # :nodoc:
       # Both Sinatra 2.0+ and Sidekiq support this term.
       # RAILS_ENV and RACK_ENV are there for legacy support.
       @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      config[:environment] = @environment
     end
 
     def symbolize_keys_deep!(hash)
@@ -396,7 +399,7 @@ module Sidekiq # :nodoc:
     end
 
     def parse_config(path)
-      erb = ERB.new(File.read(path))
+      erb = ERB.new(File.read(path), trim_mode: "-")
       erb.filename = File.expand_path(path)
       opts = YAML.safe_load(erb.result, permitted_classes: [Symbol], aliases: true) || {}
 
@@ -420,3 +423,4 @@ end
 
 require "sidekiq/systemd"
 require "sidekiq/metrics/tracking"
+require "sidekiq/job/interrupt_handler"

data/lib/sidekiq/client.rb

@@ -58,6 +58,23 @@ module Sidekiq
       end
     end
 
+    # Cancel the IterableJob with the given JID.
+    # **NB: Cancellation is asynchronous.** Iteration checks every
+    # five seconds so this will not immediately stop the given job.
+    def cancel!(jid)
+      key = "it-#{jid}"
+      _, result, _ = Sidekiq.redis do |c|
+        c.pipelined do |p|
+          p.hsetnx(key, "cancelled", Time.now.to_i)
+          p.hget(key, "cancelled")
+          p.expire(key, Sidekiq::Job::Iterable::STATE_TTL)
+          # TODO When Redis 7.2 is required
+          # p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
+        end
+      end
+      result.to_i
+    end
+
     ##
     # The main method used to push a job to Redis.  Accepts a number of options:
     #
@@ -66,6 +83,7 @@ module Sidekiq
     #   args - an array of simple arguments to the perform method, must be JSON-serializable
     #   at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
     #   retry - whether to retry this job if it fails, default true or an integer number of retries
+    #   retry_for - relative amount of time to retry this job if it fails, default nil
     #   backtrace - whether to save any error backtrace, default false
     #
     # If class is set to the class name, the jobs' options will be based on Sidekiq's default
@@ -73,7 +91,7 @@ module Sidekiq
     #
     # Any options valid for a job class's sidekiq_options are also available here.
     #
-    # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
+    # All keys must be strings, not symbols.  NB: because we are serializing to JSON, all
     # symbols in 'args' will be converted to strings.  Note that +backtrace: true+ can take quite a bit of
     # space in Redis; a large volume of failing jobs can start Redis swapping if you aren't careful.
     #
@@ -96,8 +114,9 @@ module Sidekiq
 
     ##
     # Push a large number of jobs to Redis. This method cuts out the redis
-    # network round trip latency.  I wouldn't recommend pushing more than
-    # 1000 per call but YMMV based on network quality, size of job args, etc.
+    # network round trip latency. It pushes jobs in batches if more than
+    # `:batch_size` (1000 by default) of jobs are passed. I wouldn't recommend making `:batch_size`
+    # larger than 1000 but YMMV based on network quality, size of job args, etc.
     # A large number of jobs can cause a bit of Redis command processing latency.
     #
     # Takes the same arguments as #push except that args is expected to be
@@ -105,13 +124,15 @@ module Sidekiq
     # is run through the client middleware pipeline and each job gets its own Job ID
     # as normal.
     #
-    # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
-    # than the number given if the middleware stopped processing for one or more jobs.
+    # Returns an array of the of pushed jobs' jids, may contain nils if any client middleware
+    # prevented a job push.
+    #
+    # Example (pushing jobs in batches):
+    #   push_bulk('class' => MyJob, 'args' => (1..100_000).to_a, batch_size: 1_000)
+    #
     def push_bulk(items)
+      batch_size = items.delete(:batch_size) || items.delete("batch_size") || 1_000
       args = items["args"]
-      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
-      return [] if args.empty? # no jobs to push
-
       at = items.delete("at")
       raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
       raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
@@ -120,18 +141,28 @@ module Sidekiq
       raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
 
       normed = normalize_item(items)
-      payloads = args.map.with_index { |job_args, index|
-        copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
-        copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
-        result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
-          verify_json(copy)
-          copy
-        end
-        result || nil
-      }.compact
+      slice_index = 0
+      result = args.each_slice(batch_size).flat_map do |slice|
+        raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless slice.is_a?(Array) && slice.all?(Array)
+        break [] if slice.empty? # no jobs to push
+
+        payloads = slice.map.with_index { |job_args, index|
+          copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
+          copy["at"] = (at.is_a?(Array) ? at[slice_index + index] : at) if at
+          result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
+            verify_json(copy)
+            copy
+          end
+          result || nil
+        }
+        slice_index += batch_size
+
+        to_push = payloads.compact
+        raw_push(to_push) unless to_push.empty?
+        payloads.map { |payload| payload&.[]("jid") }
+      end
 
-      raw_push(payloads) unless payloads.empty?
-      payloads.collect { |payload| payload["jid"] }
+      result.is_a?(Enumerator::Lazy) ? result.force : result
     end
 
     # Allows sharding of jobs across any number of Redis instances.  All jobs
@@ -160,8 +191,8 @@ module Sidekiq
         new.push(item)
       end
 
-      def push_bulk(items)
-        new.push_bulk(items)
+      def push_bulk(...)
+        new.push_bulk(...)
       end
 
       # Resque compatibility helpers.  Note all helpers
@@ -234,7 +265,12 @@ module Sidekiq
     def atomic_push(conn, payloads)
       if payloads.first.key?("at")
         conn.zadd("schedule", payloads.flat_map { |hash|
-          at = hash.delete("at").to_s
+          at = hash["at"].to_s
+          # ActiveJob sets this but the job has not been enqueued yet
+          hash.delete("enqueued_at")
+          # TODO: Use hash.except("at") when support for Ruby 2.7 is dropped
+          hash = hash.dup
+          hash.delete("at")
           [at, Sidekiq.dump_json(hash)]
         })
       else