sidekiq 7.0.0 → 7.3.0

data/lib/sidekiq/config.rb

@@ -17,6 +17,10 @@ module Sidekiq
       poll_interval_average: nil,
       average_scheduled_poll_interval: 5,
       on_complex_arguments: :raise,
+      iteration: {
+        max_job_runtime: nil,
+        retry_backoff: 0
+      },
       error_handlers: [],
       death_handlers: [],
       lifecycle_events: {
@@ -30,15 +34,18 @@ module Sidekiq
       },
       dead_max_jobs: 10_000,
       dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
-      reloader: proc { |&block| block.call }
+      reloader: proc { |&block| block.call },
+      backtrace_cleaner: ->(backtrace) { backtrace }
     }
 
-    ERROR_HANDLER = ->(ex, ctx) {
-      cfg = ctx[:_config] || Sidekiq.default_configuration
+    ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
       l = cfg.logger
       l.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
       l.warn("#{ex.class.name}: #{ex.message}")
-      l.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
+      unless ex.backtrace.nil?
+        backtrace = cfg[:backtrace_cleaner].call(ex.backtrace)
+        l.warn(backtrace.join("\n"))
+      end
     }
 
     def initialize(options = {})
@@ -49,9 +56,13 @@ module Sidekiq
       @capsules = {}
     end
 
-    def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!
+    def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
     attr_reader :capsules
 
+    def to_json(*)
+      Sidekiq.dump_json(@options)
+    end
+
     # LEGACY: edits the default capsule
     # config.concurrency = 5
     def concurrency=(val)
@@ -123,18 +134,18 @@ module Sidekiq
     private def local_redis_pool
       # this is our internal client/housekeeping pool. each capsule has its
       # own pool for executing threads.
-      @redis ||= new_redis_pool(5, "internal")
+      @redis ||= new_redis_pool(10, "internal")
     end
 
     def new_redis_pool(size, name = "unset")
       # connection pool is lazy, it will not create connections unless you actually need them
       # so don't be skimpy!
-      RedisConnection.create(@redis_config.merge(size: size, logger: logger, pool_name: name))
+      RedisConnection.create({size: size, logger: logger, pool_name: name}.merge(@redis_config))
     end
 
     def redis_info
       redis do |conn|
-        conn.info
+        conn.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
       rescue RedisClientAdapter::CommandError => ex
         # 2850 return fake version when INFO command has (probably) been renamed
         raise unless /unknown command/.match?(ex.message)
@@ -248,19 +259,28 @@ module Sidekiq
         return
       end
 
-      logger.extend(Sidekiq::LoggingUtils)
       @logger = logger
     end
 
+    private def parameter_size(handler)
+      target = handler.is_a?(Proc) ? handler : handler.method(:call)
+      target.parameters.size
+    end
+
     # INTERNAL USE ONLY
     def handle_exception(ex, ctx = {})
       if @options[:error_handlers].size == 0
         p ["!!!!!", ex]
       end
-      ctx[:_config] = self
       @options[:error_handlers].each do |handler|
-        handler.call(ex, ctx)
-      rescue => e
+        if parameter_size(handler) == 2
+          # TODO Remove in 8.0
+          logger.info { "DEPRECATION: Sidekiq exception handlers now take three arguments, see #{handler}" }
+          handler.call(ex, {_config: self}.merge(ctx))
+        else
+          handler.call(ex, ctx, self)
+        end
+      rescue Exception => e
         l = logger
         l.error "!!! ERROR HANDLER THREW AN ERROR !!!"
         l.error e

data/lib/sidekiq/deploy.rb

@@ -21,20 +21,20 @@ module Sidekiq
     }
 
     def self.mark!(label = nil)
-      label ||= LABEL_MAKER.call
-      Sidekiq::Deploy.new.mark(label: label)
+      Sidekiq::Deploy.new.mark!(label: label)
     end
 
     def initialize(pool = Sidekiq::RedisConnection.create)
       @pool = pool
     end
 
-    def mark(at: Time.now, label: "")
+    def mark!(at: Time.now, label: nil)
+      label ||= LABEL_MAKER.call
       # we need to round the timestamp so that we gracefully
       # handle an very common error in marking deploys:
       # having every process mark its deploy, leading
       # to N marks for each deploy. Instead we round the time
-      # to the minute so that multple marks within that minute
+      # to the minute so that multiple marks within that minute
       # will all naturally rollup into one mark per minute.
       whence = at.utc
       floor = Time.utc(whence.year, whence.month, whence.mday, whence.hour, whence.min, 0)
@@ -44,7 +44,7 @@ module Sidekiq
 
       @pool.with do |c|
         # only allow one deploy mark for a given label for the next minute
-        lock = c.set("deploylock-#{label}", stamp, nx: true, ex: 60)
+        lock = c.set("deploylock-#{label}", stamp, "nx", "ex", "60")
         if lock
           c.multi do |pipe|
             pipe.hsetnx(key, stamp, label)

data/lib/sidekiq/embedded.rb

@@ -15,9 +15,9 @@ module Sidekiq
       fire_event(:startup, reverse: false, reraise: true)
       @launcher = Sidekiq::Launcher.new(@config, embedded: true)
       @launcher.run
-      sleep 0.1 # pause to give threads time to spin up
+      sleep 0.2 # pause to give threads time to spin up
 
-      logger.info "Embedded mode running with #{Thread.list.size} threads"
+      logger.info "Sidekiq running embedded, total process thread count: #{Thread.list.size}"
       logger.debug { Thread.list.map(&:name) }
     end
 
@@ -49,7 +49,7 @@ module Sidekiq
 
           WARNING: Your Redis instance will evict Sidekiq data under heavy load.
           The 'noeviction' maxmemory policy is recommended (current policy: '#{maxmemory_policy}').
-          See: https://github.com/mperham/sidekiq/wiki/Using-Redis#memory
+          See: https://github.com/sidekiq/sidekiq/wiki/Using-Redis#memory
 
         EOM
       end

data/lib/sidekiq/fetch.rb

@@ -30,11 +30,9 @@ module Sidekiq # :nodoc:
     def initialize(cap)
       raise ArgumentError, "missing queue list" unless cap.queues
       @config = cap
-      @strictly_ordered_queues = (config.queues.size == config.queues.uniq.size)
+      @strictly_ordered_queues = cap.mode == :strict
       @queues = config.queues.map { |q| "queue:#{q}" }
-      if @strictly_ordered_queues
-        @queues.uniq!
-      end
+      @queues.uniq! if @strictly_ordered_queues
     end
 
     def retrieve_work
@@ -46,7 +44,7 @@ module Sidekiq # :nodoc:
         return nil
       end
 
-      queue, job = redis { |conn| conn.blocking_call(false, "brpop", *qs, TIMEOUT) }
+      queue, job = redis { |conn| conn.blocking_call(TIMEOUT, "brpop", *qs, TIMEOUT) }
       UnitOfWork.new(queue, job, config) if queue
     end
 

data/lib/sidekiq/iterable_job.rb

@@ -0,0 +1,53 @@
+require "sidekiq/job/iterable"
+
+# Iterable jobs are ones which provide a sequence to process using
+# `build_enumerator(*args, cursor: cursor)` and then process each
+# element of that sequence in `each_iteration(item, *args)`.
+#
+# The job is kicked off as normal:
+#
+#     ProcessUserSet.perform_async(123)
+#
+# but instead of calling `perform`, Sidekiq will call:
+#
+#     enum = ProcessUserSet#build_enumerator(123, cursor:nil)
+#
+# Your Enumerator must yield `(object, updated_cursor)` and
+# Sidekiq will call your `each_iteration` method:
+#
+#     ProcessUserSet#each_iteration(object, 123)
+#
+# After every iteration, Sidekiq will check for shutdown. If we are
+# stopping, the cursor will be saved to Redis and the job re-queued
+# to pick up the rest of the work upon restart. Your job will get
+# the updated_cursor so it can pick up right where it stopped.
+#
+#     enum = ProcessUserSet#build_enumerator(123, cursor: updated_cursor)
+#
+# The cursor object must be serializable to JSON.
+#
+# Note there are several APIs to help you build enumerators for
+# ActiveRecord Relations, CSV files, etc. See sidekiq/job/iterable/*.rb.
+module Sidekiq
+  module IterableJob
+    def self.included(base)
+      base.include Sidekiq::Job
+      base.include Sidekiq::Job::Iterable
+    end
+
+    # def build_enumerator(*args, cursor:)
+    # def each_iteration(item, *args)
+
+    # Your job can also define several callbacks during points
+    # in each job's lifecycle.
+    #
+    # def on_start
+    # def on_resume
+    # def on_stop
+    # def on_complete
+    # def around_iteration
+    #
+    # To keep things simple and compatible, this is the same
+    # API as the `sidekiq-iteration` gem.
+  end
+end

data/lib/sidekiq/job/interrupt_handler.rb

@@ -0,0 +1,22 @@
+module Sidekiq
+  module Job
+    class InterruptHandler
+      include Sidekiq::ServerMiddleware
+
+      def call(instance, hash, queue)
+        yield
+      rescue Interrupted
+        logger.debug "Interrupted, re-queueing..."
+        c = Sidekiq::Client.new
+        c.push(hash)
+        raise Sidekiq::JobRetry::Skip
+      end
+    end
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Job::InterruptHandler
+  end
+end

data/lib/sidekiq/job/iterable/active_record_enumerator.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Job
+    module Iterable
+      # @api private
+      class ActiveRecordEnumerator
+        def initialize(relation, cursor: nil, **options)
+          @relation = relation
+          @cursor = cursor
+          @options = options
+        end
+
+        def records
+          Enumerator.new(-> { @relation.count }) do |yielder|
+            @relation.find_each(**@options, start: @cursor) do |record|
+              yielder.yield(record, record.id)
+            end
+          end
+        end
+
+        def batches
+          Enumerator.new(-> { @relation.count }) do |yielder|
+            @relation.find_in_batches(**@options, start: @cursor) do |batch|
+              yielder.yield(batch, batch.last.id)
+            end
+          end
+        end
+
+        def relations
+          Enumerator.new(-> { relations_size }) do |yielder|
+            # Convenience to use :batch_size for all the
+            # ActiveRecord batching methods.
+            options = @options.dup
+            options[:of] ||= options.delete(:batch_size)
+
+            @relation.in_batches(**options, start: @cursor) do |relation|
+              last_record = relation.last
+              yielder.yield(relation, last_record.id)
+            end
+          end
+        end
+
+        private
+
+        def relations_size
+          batch_size = @options[:batch_size] || 1000
+          (@relation.count + batch_size - 1) / batch_size # ceiling division
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/job/iterable/csv_enumerator.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Job
+    module Iterable
+      # @api private
+      class CsvEnumerator
+        def initialize(csv)
+          unless defined?(CSV) && csv.instance_of?(CSV)
+            raise ArgumentError, "CsvEnumerator.new takes CSV object"
+          end
+
+          @csv = csv
+        end
+
+        def rows(cursor:)
+          @csv.lazy
+            .each_with_index
+            .drop(cursor || 0)
+            .to_enum { count_of_rows_in_file }
+        end
+
+        def batches(cursor:, batch_size: 100)
+          @csv.lazy
+            .each_slice(batch_size)
+            .with_index
+            .drop(cursor || 0)
+            .to_enum { (count_of_rows_in_file.to_f / batch_size).ceil }
+        end
+
+        private
+
+        def count_of_rows_in_file
+          filepath = @csv.path
+          return unless filepath
+
+          count = IO.popen(["wc", "-l", filepath]) do |out|
+            out.read.strip.to_i
+          end
+
+          count -= 1 if @csv.headers
+          count
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/job/iterable/enumerators.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require_relative "active_record_enumerator"
+require_relative "csv_enumerator"
+
+module Sidekiq
+  module Job
+    module Iterable
+      module Enumerators
+        # Builds Enumerator object from a given array, using +cursor+ as an offset.
+        #
+        # @param array [Array]
+        # @param cursor [Integer] offset to start iteration from
+        #
+        # @return [Enumerator]
+        #
+        # @example
+        #   array_enumerator(['build', 'enumerator', 'from', 'any', 'array'], cursor: cursor)
+        #
+        def array_enumerator(array, cursor:)
+          raise ArgumentError, "array must be an Array" unless array.is_a?(Array)
+
+          x = array.each_with_index.drop(cursor || 0)
+          x.to_enum { x.size }
+        end
+
+        # Builds Enumerator from `ActiveRecord::Relation`.
+        # Each Enumerator tick moves the cursor one row forward.
+        #
+        # @param relation [ActiveRecord::Relation] relation to iterate
+        # @param cursor [Object] offset id to start iteration from
+        # @param options [Hash] additional options that will be passed to relevant
+        #   ActiveRecord batching methods
+        #
+        # @return [ActiveRecordEnumerator]
+        #
+        # @example
+        #   def build_enumerator(cursor:)
+        #     active_record_records_enumerator(User.all, cursor: cursor)
+        #   end
+        #
+        #   def each_iteration(user)
+        #     user.notify_about_something
+        #   end
+        #
+        def active_record_records_enumerator(relation, cursor:, **options)
+          ActiveRecordEnumerator.new(relation, cursor: cursor, **options).records
+        end
+
+        # Builds Enumerator from `ActiveRecord::Relation` and enumerates on batches of records.
+        # Each Enumerator tick moves the cursor `:batch_size` rows forward.
+        # @see #active_record_records_enumerator
+        #
+        # @example
+        #   def build_enumerator(product_id, cursor:)
+        #     active_record_batches_enumerator(
+        #       Comment.where(product_id: product_id).select(:id),
+        #       cursor: cursor,
+        #       batch_size: 100
+        #     )
+        #   end
+        #
+        #   def each_iteration(batch_of_comments, product_id)
+        #     comment_ids = batch_of_comments.map(&:id)
+        #     CommentService.call(comment_ids: comment_ids)
+        #   end
+        #
+        def active_record_batches_enumerator(relation, cursor:, **options)
+          ActiveRecordEnumerator.new(relation, cursor: cursor, **options).batches
+        end
+
+        # Builds Enumerator from `ActiveRecord::Relation` and enumerates on batches,
+        # yielding `ActiveRecord::Relation`s.
+        # @see #active_record_records_enumerator
+        #
+        # @example
+        #   def build_enumerator(product_id, cursor:)
+        #     active_record_relations_enumerator(
+        #       Product.find(product_id).comments,
+        #       cursor: cursor,
+        #       batch_size: 100,
+        #     )
+        #   end
+        #
+        #   def each_iteration(batch_of_comments, product_id)
+        #     # batch_of_comments will be a Comment::ActiveRecord_Relation
+        #     batch_of_comments.update_all(deleted: true)
+        #   end
+        #
+        def active_record_relations_enumerator(relation, cursor:, **options)
+          ActiveRecordEnumerator.new(relation, cursor: cursor, **options).relations
+        end
+
+        # Builds Enumerator from a CSV file.
+        #
+        # @param csv [CSV] an instance of CSV object
+        # @param cursor [Integer] offset to start iteration from
+        #
+        # @example
+        #   def build_enumerator(import_id, cursor:)
+        #     import = Import.find(import_id)
+        #     csv_enumerator(import.csv, cursor: cursor)
+        #   end
+        #
+        #   def each_iteration(csv_row)
+        #     # insert csv_row into database
+        #   end
+        #
+        def csv_enumerator(csv, cursor:)
+          CsvEnumerator.new(csv).rows(cursor: cursor)
+        end
+
+        # Builds Enumerator from a CSV file and enumerates on batches of records.
+        #
+        # @param csv [CSV] an instance of CSV object
+        # @param cursor [Integer] offset to start iteration from
+        # @option options :batch_size [Integer] (100) size of the batch
+        #
+        # @example
+        #   def build_enumerator(import_id, cursor:)
+        #     import = Import.find(import_id)
+        #     csv_batches_enumerator(import.csv, cursor: cursor)
+        #   end
+        #
+        #   def each_iteration(batch_of_csv_rows)
+        #     # ...
+        #   end
+        #
+        def csv_batches_enumerator(csv, cursor:, **options)
+          CsvEnumerator.new(csv).batches(cursor: cursor, **options)
+        end
+      end
+    end
+  end
+end