sidekiq 6.5.10 → 8.1.7

data/lib/sidekiq/deploy.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "sidekiq/redis_connection"
+require "time"
+
+# This file is designed to be required within the user's
+# deployment script; it should need a bare minimum of dependencies.
+# Usage:
+#
+#   require "sidekiq/deploy"
+#   Sidekiq::Deploy.mark!("Some change")
+#
+# If you do not pass a label, Sidekiq will try to use the latest
+# git commit info.
+#
+
+module Sidekiq
+  class Deploy
+    MARK_TTL = 90 * 24 * 60 * 60 # 90 days
+
+    LABEL_MAKER = -> {
+      `git log -1 --format="%h %s"`.strip
+    }
+
+    def self.mark!(label = nil)
+      Sidekiq::Deploy.new.mark!(label: label)
+    end
+
+    def initialize(pool = Sidekiq::RedisConnection.create)
+      @pool = pool
+    end
+
+    def mark!(at: Time.now, label: nil)
+      label ||= LABEL_MAKER.call
+      # we need to round the timestamp so that we gracefully
+      # handle an very common error in marking deploys:
+      # having every process mark its deploy, leading
+      # to N marks for each deploy. Instead we round the time
+      # to the minute so that multiple marks within that minute
+      # will all naturally rollup into one mark per minute.
+      whence = at.utc
+      floor = Time.utc(whence.year, whence.month, whence.mday, whence.hour, whence.min, 0)
+      datecode = floor.strftime("%Y%m%d")
+      key = "#{datecode}-marks"
+      stamp = floor.iso8601
+
+      @pool.with do |c|
+        # only allow one deploy mark for a given label for the next minute
+        lock = c.set("deploylock-#{label}", stamp, "nx", "ex", "60")
+        if lock
+          c.multi do |pipe|
+            pipe.hsetnx(key, stamp, label)
+            pipe.expire(key, MARK_TTL)
+          end
+        end
+      end
+    end
+
+    def fetch(date = Time.now.utc.to_date)
+      datecode = date.strftime("%Y%m%d")
+      @pool.with { |c| c.hgetall("#{datecode}-marks") }
+    end
+  end
+end

data/lib/sidekiq/embedded.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "sidekiq/component"
+require "sidekiq/launcher"
+require "sidekiq/metrics/tracking"
+
+module Sidekiq
+  class Embedded
+    include Sidekiq::Component
+
+    def initialize(config)
+      @config = config
+    end
+
+    def run
+      housekeeping
+      fire_event(:startup, reverse: false, reraise: true)
+      @launcher = Sidekiq::Launcher.new(@config, embedded: true)
+      @launcher.run
+      sleep 0.2 # pause to give threads time to spin up
+
+      logger.info "Sidekiq running embedded, total process thread count: #{Thread.list.size}"
+      logger.debug { Thread.list.map(&:name) }
+    end
+
+    def quiet
+      @launcher&.quiet
+    end
+
+    def stop
+      @launcher&.stop
+    end
+
+    private
+
+    def housekeeping
+      @config[:tag] ||= default_tag
+      logger.info "Running in #{RUBY_DESCRIPTION}"
+      logger.info Sidekiq::LICENSE
+      logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
+
+      # touch the connection pool so it is created before we
+      # fire startup and start multithreading.
+      info = config.redis_info
+      ver = Gem::Version.new(info["redis_version"])
+      raise "You are connected to Redis #{ver}, Sidekiq requires Redis 7.0.0 or greater" if ver < Gem::Version.new("7.0.0")
+
+      maxmemory_policy = info["maxmemory_policy"]
+      if maxmemory_policy != "noeviction"
+        logger.warn <<~EOM
+
+
+          WARNING: Your Redis instance will evict Sidekiq data under heavy load.
+          The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
+          See: https://github.com/sidekiq/sidekiq/wiki/Using-Redis#memory
+
+        EOM
+      end
+
+      logger.debug { "Client Middleware: #{@config.default_capsule.client_middleware.map(&:klass).join(", ")}" }
+      logger.debug { "Server Middleware: #{@config.default_capsule.server_middleware.map(&:klass).join(", ")}" }
+    end
+  end
+end

data/lib/sidekiq/fetch.rb

@@ -2,10 +2,12 @@
 
 require "sidekiq"
 require "sidekiq/component"
+require "sidekiq/capsule"
 
 module Sidekiq # :nodoc:
   class BasicFetch
     include Sidekiq::Component
+
     # We want the fetch operation to timeout every few seconds so the thread
     # can check if the process is shutting down.
     TIMEOUT = 2
@@ -26,31 +28,28 @@ module Sidekiq # :nodoc:
       end
     }
 
-    def initialize(config)
-      raise ArgumentError, "missing queue list" unless config[:queues]
-      @config = config
-      @strictly_ordered_queues = !!@config[:strict]
-      @queues = @config[:queues].map { |q| "queue:#{q}" }
-      if @strictly_ordered_queues
-        @queues.uniq!
-        @queues << {timeout: TIMEOUT}
-      end
+    def initialize(cap)
+      raise ArgumentError, "missing queue list" unless cap.queues
+      @config = cap
+      @strictly_ordered_queues = cap.mode == :strict
+      @queues = config.queues.map { |q| "queue:#{q}" }
+      @queues.uniq! if @strictly_ordered_queues
     end
 
     def retrieve_work
       qs = queues_cmd
       # 4825 Sidekiq Pro with all queues paused will return an
-      # empty set of queues with a trailing TIMEOUT value.
-      if qs.size <= 1
+      # empty set of queues
+      if qs.size <= 0
         sleep(TIMEOUT)
         return nil
       end
 
-      queue, job = redis { |conn| conn.brpop(*qs) }
+      queue, job = redis { |conn| conn.blocking_call(TIMEOUT, "brpop", *qs, TIMEOUT) }
       UnitOfWork.new(queue, job, config) if queue
     end
 
-    def bulk_requeue(inprogress, options)
+    def bulk_requeue(inprogress)
       return if inprogress.empty?
 
       logger.debug { "Re-queueing terminated jobs" }
@@ -83,7 +82,6 @@ module Sidekiq # :nodoc:
       else
         permute = @queues.shuffle
         permute.uniq!
-        permute << {timeout: TIMEOUT}
         permute
       end
     end

data/lib/sidekiq/iterable_job.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "sidekiq/job/iterable"
+
+# Iterable jobs are ones which provide a sequence to process using
+# `build_enumerator(*args, cursor: cursor)` and then process each
+# element of that sequence in `each_iteration(item, *args)`.
+#
+# The job is kicked off as normal:
+#
+#     ProcessUserSet.perform_async(123)
+#
+# but instead of calling `perform`, Sidekiq will call:
+#
+#     enum = ProcessUserSet#build_enumerator(123, cursor:nil)
+#
+# Your Enumerator must yield `(object, updated_cursor)` and
+# Sidekiq will call your `each_iteration` method:
+#
+#     ProcessUserSet#each_iteration(object, 123)
+#
+# After every iteration, Sidekiq will check for shutdown. If we are
+# stopping, the cursor will be saved to Redis and the job re-queued
+# to pick up the rest of the work upon restart. Your job will get
+# the updated_cursor so it can pick up right where it stopped.
+#
+#     enum = ProcessUserSet#build_enumerator(123, cursor: updated_cursor)
+#
+# The cursor object must be serializable to JSON.
+#
+# Note there are several APIs to help you build enumerators for
+# ActiveRecord Relations, CSV files, etc. See sidekiq/job/iterable/*.rb.
+module Sidekiq
+  module IterableJob
+    def self.included(base)
+      base.include Sidekiq::Job
+      base.include Sidekiq::Job::Iterable
+    end
+
+    # def build_enumerator(*args, cursor:)
+    # def each_iteration(item, *args)
+
+    # Your job can also define several callbacks during points
+    # in each job's lifecycle.
+    #
+    # def on_start
+    # def on_resume
+    # def on_stop
+    # def on_cancel
+    # def on_complete
+    # def around_iteration
+    #
+    # To keep things simple and compatible, this is the same
+    # API as the `sidekiq-iteration` gem.
+  end
+end

data/lib/sidekiq/job/interrupt_handler.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Job
+    class InterruptHandler
+      include Sidekiq::ServerMiddleware
+
+      def call(instance, hash, queue)
+        yield
+      rescue Interrupted
+        logger.debug "Interrupted, re-queueing..."
+        c = Sidekiq::Client.new
+        c.push(hash)
+        raise Sidekiq::JobRetry::Skip
+      end
+    end
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Job::InterruptHandler
+  end
+end

data/lib/sidekiq/job/iterable/active_record_enumerator.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Job
+    module Iterable
+      # @api private
+      class ActiveRecordEnumerator
+        def initialize(relation, cursor: nil, **options)
+          @relation = relation
+          @cursor = cursor
+          @options = options
+        end
+
+        def records
+          Enumerator.new(-> { @relation.count }) do |yielder|
+            @relation.find_each(**@options, start: @cursor) do |record|
+              yielder.yield(record, record.id)
+            end
+          end
+        end
+
+        def batches
+          Enumerator.new(-> { @relation.count }) do |yielder|
+            @relation.find_in_batches(**@options, start: @cursor) do |batch|
+              yielder.yield(batch, batch.first.id)
+            end
+          end
+        end
+
+        def relations
+          Enumerator.new(-> { relations_size }) do |yielder|
+            # Convenience to use :batch_size for all the
+            # ActiveRecord batching methods.
+            options = @options.dup
+            options[:of] ||= options.delete(:batch_size)
+
+            @relation.in_batches(**options, start: @cursor) do |relation|
+              first_record = relation.first
+              yielder.yield(relation, first_record.id)
+            end
+          end
+        end
+
+        private
+
+        def relations_size
+          batch_size = @options[:batch_size] || 1000
+          (@relation.count + batch_size - 1) / batch_size # ceiling division
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/job/iterable/csv_enumerator.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Job
+    module Iterable
+      # @api private
+      class CsvEnumerator
+        def initialize(csv)
+          unless defined?(CSV) && csv.instance_of?(CSV)
+            raise ArgumentError, "CsvEnumerator.new takes CSV object"
+          end
+
+          @csv = csv
+        end
+
+        def rows(cursor:)
+          @csv.lazy
+            .each_with_index
+            .drop(cursor || 0)
+            .to_enum { count_of_rows_in_file }
+        end
+
+        def batches(cursor:, batch_size: 100)
+          @csv.lazy
+            .each_slice(batch_size)
+            .with_index
+            .drop(cursor || 0)
+            .to_enum { (count_of_rows_in_file.to_f / batch_size).ceil }
+        end
+
+        private
+
+        def count_of_rows_in_file
+          filepath = @csv.path
+          return unless filepath
+
+          count = IO.popen(["wc", "-l", filepath]) do |out|
+            out.read.strip.to_i
+          end
+
+          count -= 1 if @csv.headers
+          count
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/job/iterable/enumerators.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require_relative "active_record_enumerator"
+require_relative "csv_enumerator"
+
+module Sidekiq
+  module Job
+    module Iterable
+      module Enumerators
+        # Builds Enumerator object from a given array, using +cursor+ as an offset.
+        #
+        # @param array [Array]
+        # @param cursor [Integer] offset to start iteration from
+        #
+        # @return [Enumerator]
+        #
+        # @example
+        #   array_enumerator(['build', 'enumerator', 'from', 'any', 'array'], cursor: cursor)
+        #
+        def array_enumerator(array, cursor:)
+          raise ArgumentError, "array must be an Array" unless array.is_a?(Array)
+
+          x = array.each_with_index.drop(cursor || 0)
+          x.to_enum { x.size }
+        end
+
+        # Builds Enumerator from `ActiveRecord::Relation`.
+        # Each Enumerator tick moves the cursor one row forward.
+        #
+        # @param relation [ActiveRecord::Relation] relation to iterate
+        # @param cursor [Object] offset id to start iteration from
+        # @param options [Hash] additional options that will be passed to relevant
+        #   ActiveRecord batching methods
+        #
+        # @return [ActiveRecordEnumerator]
+        #
+        # @example
+        #   def build_enumerator(cursor:)
+        #     active_record_records_enumerator(User.all, cursor: cursor)
+        #   end
+        #
+        #   def each_iteration(user)
+        #     user.notify_about_something
+        #   end
+        #
+        def active_record_records_enumerator(relation, cursor:, **options)
+          ActiveRecordEnumerator.new(relation, cursor: cursor, **options).records
+        end
+
+        # Builds Enumerator from `ActiveRecord::Relation` and enumerates on batches of records.
+        # Each Enumerator tick moves the cursor `:batch_size` rows forward.
+        # @see #active_record_records_enumerator
+        #
+        # @example
+        #   def build_enumerator(product_id, cursor:)
+        #     active_record_batches_enumerator(
+        #       Comment.where(product_id: product_id).select(:id),
+        #       cursor: cursor,
+        #       batch_size: 100
+        #     )
+        #   end
+        #
+        #   def each_iteration(batch_of_comments, product_id)
+        #     comment_ids = batch_of_comments.map(&:id)
+        #     CommentService.call(comment_ids: comment_ids)
+        #   end
+        #
+        def active_record_batches_enumerator(relation, cursor:, **options)
+          ActiveRecordEnumerator.new(relation, cursor: cursor, **options).batches
+        end
+
+        # Builds Enumerator from `ActiveRecord::Relation` and enumerates on batches,
+        # yielding `ActiveRecord::Relation`s.
+        # @see #active_record_records_enumerator
+        #
+        # @example
+        #   def build_enumerator(product_id, cursor:)
+        #     active_record_relations_enumerator(
+        #       Product.find(product_id).comments,
+        #       cursor: cursor,
+        #       batch_size: 100,
+        #     )
+        #   end
+        #
+        #   def each_iteration(batch_of_comments, product_id)
+        #     # batch_of_comments will be a Comment::ActiveRecord_Relation
+        #     batch_of_comments.update_all(deleted: true)
+        #   end
+        #
+        def active_record_relations_enumerator(relation, cursor:, **options)
+          ActiveRecordEnumerator.new(relation, cursor: cursor, **options).relations
+        end
+
+        # Builds Enumerator from a CSV file.
+        #
+        # @param csv [CSV] an instance of CSV object
+        # @param cursor [Integer] offset to start iteration from
+        #
+        # @example
+        #   def build_enumerator(import_id, cursor:)
+        #     import = Import.find(import_id)
+        #     csv_enumerator(import.csv, cursor: cursor)
+        #   end
+        #
+        #   def each_iteration(csv_row)
+        #     # insert csv_row into database
+        #   end
+        #
+        def csv_enumerator(csv, cursor:)
+          CsvEnumerator.new(csv).rows(cursor: cursor)
+        end
+
+        # Builds Enumerator from a CSV file and enumerates on batches of records.
+        #
+        # @param csv [CSV] an instance of CSV object
+        # @param cursor [Integer] offset to start iteration from
+        # @option options :batch_size [Integer] (100) size of the batch
+        #
+        # @example
+        #   def build_enumerator(import_id, cursor:)
+        #     import = Import.find(import_id)
+        #     csv_batches_enumerator(import.csv, cursor: cursor)
+        #   end
+        #
+        #   def each_iteration(batch_of_csv_rows)
+        #     # ...
+        #   end
+        #
+        def csv_batches_enumerator(csv, cursor:, **options)
+          CsvEnumerator.new(csv).batches(cursor: cursor, **options)
+        end
+      end
+    end
+  end
+end