sidekiq 6.1.0 → 7.0.0

data/lib/sidekiq/embedded.rb

@@ -0,0 +1,61 @@
+require "sidekiq/component"
+require "sidekiq/launcher"
+require "sidekiq/metrics/tracking"
+
+module Sidekiq
+  class Embedded
+    include Sidekiq::Component
+
+    def initialize(config)
+      @config = config
+    end
+
+    def run
+      housekeeping
+      fire_event(:startup, reverse: false, reraise: true)
+      @launcher = Sidekiq::Launcher.new(@config, embedded: true)
+      @launcher.run
+      sleep 0.1 # pause to give threads time to spin up
+
+      logger.info "Embedded mode running with #{Thread.list.size} threads"
+      logger.debug { Thread.list.map(&:name) }
+    end
+
+    def quiet
+      @launcher&.quiet
+    end
+
+    def stop
+      @launcher&.stop
+    end
+
+    private
+
+    def housekeeping
+      logger.info "Running in #{RUBY_DESCRIPTION}"
+      logger.info Sidekiq::LICENSE
+      logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
+
+      # touch the connection pool so it is created before we
+      # fire startup and start multithreading.
+      info = config.redis_info
+      ver = Gem::Version.new(info["redis_version"])
+      raise "You are connecting to Redis #{ver}, Sidekiq requires Redis 6.2.0 or greater" if ver < Gem::Version.new("6.2.0")
+
+      maxmemory_policy = info["maxmemory_policy"]
+      if maxmemory_policy != "noeviction"
+        logger.warn <<~EOM
+
+
+          WARNING: Your Redis instance will evict Sidekiq data under heavy load.
+          The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
+          See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
+
+        EOM
+      end
+
+      logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
+      logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
+    end
+  end
+end

data/lib/sidekiq/fetch.rb

@@ -1,14 +1,17 @@
 # frozen_string_literal: true
 
 require "sidekiq"
+require "sidekiq/component"
+require "sidekiq/capsule"
 
-module Sidekiq
+module Sidekiq # :nodoc:
   class BasicFetch
+    include Sidekiq::Component
     # We want the fetch operation to timeout every few seconds so the thread
     # can check if the process is shutting down.
     TIMEOUT = 2
 
-    UnitOfWork = Struct.new(:queue, :job) {
+    UnitOfWork = Struct.new(:queue, :job, :config) {
       def acknowledge
         # nothing to do
       end
@@ -18,50 +21,55 @@ module Sidekiq
       end
 
       def requeue
-        Sidekiq.redis do |conn|
+        config.redis do |conn|
           conn.rpush(queue, job)
         end
       end
     }
 
-    def initialize(options)
-      raise ArgumentError, "missing queue list" unless options[:queues]
-      @options = options
-      @strictly_ordered_queues = !!@options[:strict]
-      @queues = @options[:queues].map { |q| "queue:#{q}" }
+    def initialize(cap)
+      raise ArgumentError, "missing queue list" unless cap.queues
+      @config = cap
+      @strictly_ordered_queues = (config.queues.size == config.queues.uniq.size)
+      @queues = config.queues.map { |q| "queue:#{q}" }
       if @strictly_ordered_queues
         @queues.uniq!
-        @queues << TIMEOUT
       end
     end
 
     def retrieve_work
-      work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
-      UnitOfWork.new(*work) if work
+      qs = queues_cmd
+      # 4825 Sidekiq Pro with all queues paused will return an
+      # empty set of queues
+      if qs.size <= 0
+        sleep(TIMEOUT)
+        return nil
+      end
+
+      queue, job = redis { |conn| conn.blocking_call(false, "brpop", *qs, TIMEOUT) }
+      UnitOfWork.new(queue, job, config) if queue
     end
 
-    # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
-    # an instance method will make it async to the Fetcher actor
-    def bulk_requeue(inprogress, options)
+    def bulk_requeue(inprogress)
       return if inprogress.empty?
 
-      Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+      logger.debug { "Re-queueing terminated jobs" }
       jobs_to_requeue = {}
       inprogress.each do |unit_of_work|
         jobs_to_requeue[unit_of_work.queue] ||= []
         jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
       end
 
-      Sidekiq.redis do |conn|
-        conn.pipelined do
+      redis do |conn|
+        conn.pipelined do |pipeline|
           jobs_to_requeue.each do |queue, jobs|
-            conn.rpush(queue, jobs)
+            pipeline.rpush(queue, jobs)
           end
         end
       end
-      Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
+      logger.info("Pushed #{inprogress.size} jobs back to Redis")
     rescue => ex
-      Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
+      logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
     end
 
     # Creating the Redis#brpop command takes into account any
@@ -73,9 +81,9 @@ module Sidekiq
       if @strictly_ordered_queues
         @queues
       else
-        queues = @queues.shuffle!.uniq
-        queues << TIMEOUT
-        queues
+        permute = @queues.shuffle
+        permute.uniq!
+        permute
       end
     end
   end

data/lib/sidekiq/{worker.rb → job.rb}

@@ -4,11 +4,12 @@ require "sidekiq/client"
 
 module Sidekiq
   ##
-  # Include this module in your worker class and you can easily create
+  # Include this module in your job class and you can easily create
   # asynchronous jobs:
   #
-  #   class HardWorker
-  #     include Sidekiq::Worker
+  #   class HardJob
+  #     include Sidekiq::Job
+  #     sidekiq_options queue: 'critical', retry: 5
   #
   #     def perform(*args)
   #       # do some work
@@ -17,10 +18,30 @@ module Sidekiq
   #
   # Then in your Rails app, you can do this:
   #
-  #   HardWorker.perform_async(1, 2, 3)
+  #   HardJob.perform_async(1, 2, 3)
   #
   # Note that perform_async is a class method, perform is an instance method.
-  module Worker
+  #
+  # Sidekiq::Job also includes several APIs to provide compatibility with
+  # ActiveJob.
+  #
+  #   class SomeJob
+  #     include Sidekiq::Job
+  #     queue_as :critical
+  #
+  #     def perform(...)
+  #     end
+  #   end
+  #
+  #   SomeJob.set(wait_until: 1.hour).perform_async(123)
+  #
+  # Note that arguments passed to the job must still obey Sidekiq's
+  # best practice for simple, JSON-native data types. Sidekiq will not
+  # implement ActiveJob's more complex argument serialization. For
+  # this reason, we don't implement `perform_later` as our call semantics
+  # are very different.
+  #
+  module Job
     ##
     # The Options module is extracted so we can include it in ActiveJob::Base
     # and allow native AJs to configure Sidekiq features/internals.
@@ -36,11 +57,11 @@ module Sidekiq
         ACCESSOR_MUTEX = Mutex.new
 
         ##
-        # Allows customization for this type of Worker.
+        # Allows customization for this type of Job.
         # Legal options:
         #
         #   queue - name of queue to use for this job type, default *default*
-        #   retry - enable retries for this Worker in case of error during execution,
+        #   retry - enable retries for this Job in case of error during execution,
         #      *true* to use the default or *Integer* count
         #   backtrace - whether to save any error backtrace in the retry payload to display in web UI,
         #      can be true, false or an integer number of lines to save, default *false*
@@ -61,7 +82,7 @@ module Sidekiq
         end
 
         def get_sidekiq_options # :nodoc:
-          self.sidekiq_options_hash ||= Sidekiq.default_worker_options
+          self.sidekiq_options_hash ||= Sidekiq.default_job_options
         end
 
         def sidekiq_class_attribute(*attrs)
@@ -135,7 +156,7 @@ module Sidekiq
     attr_accessor :jid
 
     def self.included(base)
-      raise ArgumentError, "Sidekiq::Worker cannot be included in an ActiveJob: #{base.name}" if base.ancestors.any? { |c| c.name == "ActiveJob::Base" }
+      raise ArgumentError, "Sidekiq::Job cannot be included in an ActiveJob: #{base.name}" if base.ancestors.any? { |c| c.name == "ActiveJob::Base" }
 
       base.include(Options)
       base.extend(ClassMethods)
@@ -147,49 +168,118 @@ module Sidekiq
 
     # This helper class encapsulates the set options for `set`, e.g.
     #
-    #     SomeWorker.set(queue: 'foo').perform_async(....)
+    #     SomeJob.set(queue: 'foo').perform_async(....)
     #
     class Setter
+      include Sidekiq::JobUtil
+
       def initialize(klass, opts)
         @klass = klass
-        @opts = opts
+        # NB: the internal hash always has stringified keys
+        @opts = opts.transform_keys(&:to_s)
+
+        # ActiveJob compatibility
+        interval = @opts.delete("wait_until") || @opts.delete("wait")
+        at(interval) if interval
       end
 
       def set(options)
-        @opts.merge!(options)
+        hash = options.transform_keys(&:to_s)
+        interval = hash.delete("wait_until") || @opts.delete("wait")
+        @opts.merge!(hash)
+        at(interval) if interval
         self
       end
 
       def perform_async(*args)
-        @klass.client_push(@opts.merge("args" => args, "class" => @klass))
+        if @opts["sync"] == true
+          perform_inline(*args)
+        else
+          @klass.client_push(@opts.merge("args" => args, "class" => @klass))
+        end
+      end
+
+      # Explicit inline execution of a job. Returns nil if the job did not
+      # execute, true otherwise.
+      def perform_inline(*args)
+        raw = @opts.merge("args" => args, "class" => @klass)
+
+        # validate and normalize payload
+        item = normalize_item(raw)
+        queue = item["queue"]
+
+        # run client-side middleware
+        cfg = Sidekiq.default_configuration
+        result = cfg.client_middleware.invoke(item["class"], item, queue, cfg.redis_pool) do
+          item
+        end
+        return nil unless result
+
+        # round-trip the payload via JSON
+        msg = Sidekiq.load_json(Sidekiq.dump_json(item))
+
+        # prepare the job instance
+        klass = Object.const_get(msg["class"])
+        job = klass.new
+        job.jid = msg["jid"]
+        job.bid = msg["bid"] if job.respond_to?(:bid)
+
+        # run the job through server-side middleware
+        result = cfg.server_middleware.invoke(job, msg, msg["queue"]) do
+          # perform it
+          job.perform(*msg["args"])
+          true
+        end
+        return nil unless result
+        # jobs do not return a result. they should store any
+        # modified state.
+        true
+      end
+      alias_method :perform_sync, :perform_inline
+
+      def perform_bulk(args, batch_size: 1_000)
+        client = @klass.build_client
+        result = args.each_slice(batch_size).flat_map do |slice|
+          client.push_bulk(@opts.merge("class" => @klass, "args" => slice))
+        end
+
+        result.is_a?(Enumerator::Lazy) ? result.force : result
       end
 
       # +interval+ must be a timestamp, numeric or something that acts
       #   numeric (like an activesupport time interval).
       def perform_in(interval, *args)
+        at(interval).perform_async(*args)
+      end
+      alias_method :perform_at, :perform_in
+
+      private
+
+      def at(interval)
         int = interval.to_f
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
-
-        payload = @opts.merge("class" => @klass, "args" => args)
         # Optimization to enqueue something now that is scheduled to go out now or in the past
-        payload["at"] = ts if ts > now
-        @klass.client_push(payload)
+        @opts["at"] = ts if ts > now
+        self
       end
-      alias_method :perform_at, :perform_in
     end
 
     module ClassMethods
       def delay(*args)
-        raise ArgumentError, "Do not call .delay on a Sidekiq::Worker class, call .perform_async"
+        raise ArgumentError, "Do not call .delay on a Sidekiq::Job class, call .perform_async"
       end
 
       def delay_for(*args)
-        raise ArgumentError, "Do not call .delay_for on a Sidekiq::Worker class, call .perform_in"
+        raise ArgumentError, "Do not call .delay_for on a Sidekiq::Job class, call .perform_in"
       end
 
       def delay_until(*args)
-        raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
+        raise ArgumentError, "Do not call .delay_until on a Sidekiq::Job class, call .perform_at"
+      end
+
+      def queue_as(q)
+        sidekiq_options("queue" => q.to_s)
       end
 
       def set(options)
@@ -197,7 +287,37 @@ module Sidekiq
       end
 
       def perform_async(*args)
-        client_push("class" => self, "args" => args)
+        Setter.new(self, {}).perform_async(*args)
+      end
+
+      # Inline execution of job's perform method after passing through Sidekiq.client_middleware and Sidekiq.server_middleware
+      def perform_inline(*args)
+        Setter.new(self, {}).perform_inline(*args)
+      end
+      alias_method :perform_sync, :perform_inline
+
+      ##
+      # Push a large number of jobs to Redis, while limiting the batch of
+      # each job payload to 1,000. This method helps cut down on the number
+      # of round trips to Redis, which can increase the performance of enqueueing
+      # large numbers of jobs.
+      #
+      # +items+ must be an Array of Arrays.
+      #
+      # For finer-grained control, use `Sidekiq::Client.push_bulk` directly.
+      #
+      # Example (3 Redis round trips):
+      #
+      #     SomeJob.perform_async(1)
+      #     SomeJob.perform_async(2)
+      #     SomeJob.perform_async(3)
+      #
+      # Would instead become (1 Redis round trip):
+      #
+      #     SomeJob.perform_bulk([[1], [2], [3]])
+      #
+      def perform_bulk(*args, **kwargs)
+        Setter.new(self, {}).perform_bulk(*args, **kwargs)
       end
 
       # +interval+ must be a timestamp, numeric or something that acts
@@ -217,11 +337,11 @@ module Sidekiq
       alias_method :perform_at, :perform_in
 
       ##
-      # Allows customization for this type of Worker.
+      # Allows customization for this type of Job.
       # Legal options:
       #
-      #   queue - use a named queue for this Worker, default 'default'
-      #   retry - enable the RetryJobs middleware for this Worker, *true* to use the default
+      #   queue - use a named queue for this Job, default 'default'
+      #   retry - enable the RetryJobs middleware for this Job, *true* to use the default
       #      or *Integer* count
       #   backtrace - whether to save any error backtrace in the retry payload to display in web UI,
       #      can be true, false or an integer number of lines to save, default *false*
@@ -234,10 +354,24 @@ module Sidekiq
       end
 
       def client_push(item) # :nodoc:
-        pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options["pool"] || Sidekiq.redis_pool
-        stringified_item = item.transform_keys(&:to_s)
+        raise ArgumentError, "Job payloads should contain no Symbols: #{item}" if item.any? { |k, v| k.is_a?(::Symbol) }
+
+        # allow the user to dynamically re-target jobs to another shard using the "pool" attribute
+        #   FooJob.set(pool: SOME_POOL).perform_async
+        old = Thread.current[:sidekiq_redis_pool]
+        pool = item.delete("pool")
+        Thread.current[:sidekiq_redis_pool] = pool if pool
+        begin
+          build_client.push(item)
+        ensure
+          Thread.current[:sidekiq_redis_pool] = old
+        end
+      end
 
-        Sidekiq::Client.new(pool).push(stringified_item)
+      def build_client # :nodoc:
+        pool = Thread.current[:sidekiq_redis_pool] || get_sidekiq_options["pool"] || Sidekiq.default_configuration.redis_pool
+        client_class = get_sidekiq_options["client_class"] || Sidekiq::Client
+        client_class.new(pool: pool)
       end
     end
   end

data/lib/sidekiq/job_logger.rb

@@ -2,7 +2,7 @@
 
 module Sidekiq
   class JobLogger
-    def initialize(logger = Sidekiq.logger)
+    def initialize(logger)
       @logger = logger
     end
 
@@ -12,46 +12,34 @@ module Sidekiq
 
       yield
 
-      with_elapsed_time_context(start) do
-        @logger.info("done")
-      end
+      Sidekiq::Context.add(:elapsed, elapsed(start))
+      @logger.info("done")
     rescue Exception
-      with_elapsed_time_context(start) do
-        @logger.info("fail")
-      end
+      Sidekiq::Context.add(:elapsed, elapsed(start))
+      @logger.info("fail")
 
       raise
     end
 
     def prepare(job_hash, &block)
-      level = job_hash["log_level"]
-      if level
-        @logger.log_at(level) do
-          Sidekiq::Context.with(job_hash_context(job_hash), &block)
-        end
-      else
-        Sidekiq::Context.with(job_hash_context(job_hash), &block)
-      end
-    end
-
-    def job_hash_context(job_hash)
       # If we're using a wrapper class, like ActiveJob, use the "wrapped"
       # attribute to expose the underlying thing.
       h = {
-        class: job_hash["wrapped"] || job_hash["class"],
+        class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"],
         jid: job_hash["jid"]
       }
-      h[:bid] = job_hash["bid"] if job_hash["bid"]
-      h[:tags] = job_hash["tags"] if job_hash["tags"]
-      h
-    end
-
-    def with_elapsed_time_context(start, &block)
-      Sidekiq::Context.with(elapsed_time_context(start), &block)
-    end
+      h[:bid] = job_hash["bid"] if job_hash.has_key?("bid")
+      h[:tags] = job_hash["tags"] if job_hash.has_key?("tags")
 
-    def elapsed_time_context(start)
-      {elapsed: elapsed(start).to_s}
+      Thread.current[:sidekiq_context] = h
+      level = job_hash["log_level"]
+      if level
+        @logger.log_at(level, &block)
+      else
+        yield
+      end
+    ensure
+      Thread.current[:sidekiq_context] = nil
     end
 
     private