sidekiq 7.2.4 → 7.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c43e6b585c25dcfc8ef8364bb36cf74f9167b981ad03faa3a8d76e0d45ebe55
-  data.tar.gz: d8c65dc03008f7280b36af94db753d4c7f68267c2eb0d78cd018322887aabbb0
+  metadata.gz: ce4653de61d5b923ce7556490e95cf6918dd1c94340417ec4093f11cc8ceda39
+  data.tar.gz: e9faa23d4535a5b647c5dc86c28ce64317ea964f44faa9141b5a883f77b80dcd
 SHA512:
-  metadata.gz: d2687692b873ab82bda2ad32e9be795150cd0a8d3d330bc19f5b509ba729bef33189e06ebac86b1906c2682187391d6cf0d532e47d03fcbea83058109c5816ef
-  data.tar.gz: 431a482baeb03fc4de50fbdfba8717fc332a9d6564fde98a77699a7bd174fa3194431385951cf689c64e04853039c95fcf287084f283e8d381b3b37d5bc665e0
+  metadata.gz: 5fa259f415261689936b4f03f179198ea6849fbe63b932b5836fa800714055ce574a80a56462355a92d125440362957ebf02e5bafbde55fe8211805c80573ae5
+  data.tar.gz: 0aaf341ae13c377e6f944344e0feef246e3450ae9eb0d74fcbf8e932e9c9fcf0d8f0dd56ce374064d4f51c4d99f4c306560c3283db27f70ae6aa5af346d83ec3

data/Changes.md

@@ -2,6 +2,57 @@
 
 [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
 
+7.3.2
+----------
+
+- Adjust ActiveRecord batch iteration to restart an interrupted batch from the beginning.
+  Each batch should be processed as a single transaction in order to be idempotent. [#6405]
+- Fix typo in S::DeadSet#kill [#6397]
+- Fix CSS issue with bottom bar in Web UI [#6414]
+
+7.3.1
+----------
+
+- Don't count job interruptions as failures in metrics [#6386]
+- Add frozen string literal to a number of .rb files.
+- Fix frozen string error with style_tag and script_tag [#6371]
+- Fix an error on Ruby 2.7 because of usage of `Hash#except` [#6376]
+
+7.3.0
+----------
+
+- **NEW FEATURE** Add `Sidekiq::IterableJob`, iteration support for long-running jobs. [#6286, fatkodima]
+  Iterable jobs are interruptible and can restart quickly if
+  running during a deploy. You must ensure that `each_iteration`
+  doesn't take more than Sidekiq's `-t` timeout (default: 25 seconds). Iterable jobs must not implement `perform`.
+```ruby
+class ProcessArrayJob
+  include Sidekiq::IterableJob
+  def build_enumerator(*args, **kwargs)
+    array_enumerator(args, **kwargs)
+  end
+  def each_iteration(arg)
+    puts arg
+  end
+end
+ProcessArrayJob.perform_async(1, 2, 3)
+```
+See the [Iteration](//github.com/sidekiq/sidekiq/wiki/Iteration) wiki page and the RDoc in `Sidekiq::IterableJob`.
+This feature should be considered BETA until the next minor release.
+- **SECURITY** The Web UI no longer allows extensions to use `<script>`.
+  Adjust CSP to disallow inline scripts within the Web UI. Please see
+  `examples/webui-ext` for how to register Web UI extensions and use
+  dynamic CSS and JS. This will make Sidekiq immune to XSS attacks. [#6270]
+- Add config option, `:skip_default_job_logging` to disable Sidekiq's default
+  start/finish job logging. [#6200]
+- Allow `Sidekiq::Limiter.redis` to use Redis Cluster [#6288]
+- Retain CurrentAttributeѕ after inline execution [#6307]
+- Ignore non-existent CurrentAttributes attributes when restoring [#6341]
+- Raise default Redis {read,write,connect} timeouts from 1 to 3 seconds
+  to minimize ReadTimeoutErrors [#6162]
+- Add `logger` as a dependency since it will become bundled in Ruby 3.5 [#6320]
+- Ignore unsupported locales in the Web UI [#6313]
+
 7.2.4
 ----------
 

data/README.md

@@ -86,7 +86,7 @@ Useful resources:
 * Occasional announcements are made to the [@sidekiq](https://ruby.social/@sidekiq) Mastodon account.
 * The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A.
 
-Every Friday morning is Sidekiq office hour: I video chat and answer questions.
+Every Thursday morning is Sidekiq office hour: I video chat and answer questions.
 See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
 
 Contributing

data/lib/generators/sidekiq/job_generator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "rails/generators/named_base"
 
 module Sidekiq

data/lib/sidekiq/api.rb

@@ -813,6 +813,8 @@ module Sidekiq
 
     # Add the given job to the Dead set.
     # @param message [String] the job data as JSON
+    # @option opts [Boolean] :notify_failure  (true) Whether death handlers should be called
+    # @option opts [Exception] :ex (RuntimeError) An exception to pass to the death handlers
     def kill(message, opts = {})
       now = Time.now.to_f
       Sidekiq.redis do |conn|
@@ -825,10 +827,14 @@ module Sidekiq
 
       if opts[:notify_failure] != false
         job = Sidekiq.load_json(message)
-        r = RuntimeError.new("Job killed by API")
-        r.set_backtrace(caller)
+        if opts[:ex]
+          ex = opts[:ex]
+        else
+          ex = RuntimeError.new("Job killed by API")
+          ex.set_backtrace(caller)
+        end
         Sidekiq.default_configuration.death_handlers.each do |handle|
-          handle.call(job, r)
+          handle.call(job, ex)
         end
       end
       true
@@ -1199,7 +1205,7 @@ module Sidekiq
       @hsh.send(*all)
     end
 
-    def respond_to_missing?(name)
+    def respond_to_missing?(name, *args)
       @hsh.respond_to?(name)
     end
   end

data/lib/sidekiq/capsule.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "sidekiq/component"
 
 module Sidekiq
@@ -17,6 +19,7 @@ module Sidekiq
   # end
   class Capsule
     include Sidekiq::Component
+    extend Forwardable
 
     attr_reader :name
     attr_reader :queues
@@ -24,6 +27,8 @@ module Sidekiq
     attr_reader :mode
     attr_reader :weights
 
+    def_delegators :@config, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
+
     def initialize(name, config)
       @name = name
       @config = config

data/lib/sidekiq/cli.rb

@@ -423,3 +423,4 @@ end
 
 require "sidekiq/systemd"
 require "sidekiq/metrics/tracking"
+require "sidekiq/job/interrupt_handler"

data/lib/sidekiq/client.rb

@@ -248,9 +248,12 @@ module Sidekiq
     def atomic_push(conn, payloads)
       if payloads.first.key?("at")
         conn.zadd("schedule", payloads.flat_map { |hash|
-          at = hash.delete("at").to_s
+          at = hash["at"].to_s
           # ActiveJob sets this but the job has not been enqueued yet
           hash.delete("enqueued_at")
+          # TODO: Use hash.except("at") when support for Ruby 2.7 is dropped
+          hash = hash.dup
+          hash.delete("at")
           [at, Sidekiq.dump_json(hash)]
         })
       else

data/lib/sidekiq/config.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "forwardable"
 
 require "set"
@@ -17,6 +19,10 @@ module Sidekiq
       poll_interval_average: nil,
       average_scheduled_poll_interval: 5,
       on_complex_arguments: :raise,
+      iteration: {
+        max_job_runtime: nil,
+        retry_backoff: 0
+      },
       error_handlers: [],
       death_handlers: [],
       lifecycle_events: {
@@ -52,7 +58,7 @@ module Sidekiq
       @capsules = {}
     end
 
-    def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!
+    def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
     attr_reader :capsules
 
     def to_json(*)

data/lib/sidekiq/deploy.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "sidekiq/redis_connection"
 require "time"
 

data/lib/sidekiq/embedded.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "sidekiq/component"
 require "sidekiq/launcher"
 require "sidekiq/metrics/tracking"

data/lib/sidekiq/fetch.rb

@@ -44,7 +44,7 @@ module Sidekiq # :nodoc:
         return nil
       end
 
-      queue, job = redis { |conn| conn.blocking_call(conn.read_timeout + TIMEOUT, "brpop", *qs, TIMEOUT) }
+      queue, job = redis { |conn| conn.blocking_call(TIMEOUT, "brpop", *qs, TIMEOUT) }
       UnitOfWork.new(queue, job, config) if queue
     end
 

data/lib/sidekiq/iterable_job.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "sidekiq/job/iterable"
+
+# Iterable jobs are ones which provide a sequence to process using
+# `build_enumerator(*args, cursor: cursor)` and then process each
+# element of that sequence in `each_iteration(item, *args)`.
+#
+# The job is kicked off as normal:
+#
+#     ProcessUserSet.perform_async(123)
+#
+# but instead of calling `perform`, Sidekiq will call:
+#
+#     enum = ProcessUserSet#build_enumerator(123, cursor:nil)
+#
+# Your Enumerator must yield `(object, updated_cursor)` and
+# Sidekiq will call your `each_iteration` method:
+#
+#     ProcessUserSet#each_iteration(object, 123)
+#
+# After every iteration, Sidekiq will check for shutdown. If we are
+# stopping, the cursor will be saved to Redis and the job re-queued
+# to pick up the rest of the work upon restart. Your job will get
+# the updated_cursor so it can pick up right where it stopped.
+#
+#     enum = ProcessUserSet#build_enumerator(123, cursor: updated_cursor)
+#
+# The cursor object must be serializable to JSON.
+#
+# Note there are several APIs to help you build enumerators for
+# ActiveRecord Relations, CSV files, etc. See sidekiq/job/iterable/*.rb.
+module Sidekiq
+  module IterableJob
+    def self.included(base)
+      base.include Sidekiq::Job
+      base.include Sidekiq::Job::Iterable
+    end
+
+    # def build_enumerator(*args, cursor:)
+    # def each_iteration(item, *args)
+
+    # Your job can also define several callbacks during points
+    # in each job's lifecycle.
+    #
+    # def on_start
+    # def on_resume
+    # def on_stop
+    # def on_complete
+    # def around_iteration
+    #
+    # To keep things simple and compatible, this is the same
+    # API as the `sidekiq-iteration` gem.
+  end
+end

data/lib/sidekiq/job/interrupt_handler.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Job
+    class InterruptHandler
+      include Sidekiq::ServerMiddleware
+
+      def call(instance, hash, queue)
+        yield
+      rescue Interrupted
+        logger.debug "Interrupted, re-queueing..."
+        c = Sidekiq::Client.new
+        c.push(hash)
+        raise Sidekiq::JobRetry::Skip
+      end
+    end
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Job::InterruptHandler
+  end
+end

data/lib/sidekiq/job/iterable/active_record_enumerator.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Job
+    module Iterable
+      # @api private
+      class ActiveRecordEnumerator
+        def initialize(relation, cursor: nil, **options)
+          @relation = relation
+          @cursor = cursor
+          @options = options
+        end
+
+        def records
+          Enumerator.new(-> { @relation.count }) do |yielder|
+            @relation.find_each(**@options, start: @cursor) do |record|
+              yielder.yield(record, record.id)
+            end
+          end
+        end
+
+        def batches
+          Enumerator.new(-> { @relation.count }) do |yielder|
+            @relation.find_in_batches(**@options, start: @cursor) do |batch|
+              yielder.yield(batch, batch.first.id)
+            end
+          end
+        end
+
+        def relations
+          Enumerator.new(-> { relations_size }) do |yielder|
+            # Convenience to use :batch_size for all the
+            # ActiveRecord batching methods.
+            options = @options.dup
+            options[:of] ||= options.delete(:batch_size)
+
+            @relation.in_batches(**options, start: @cursor) do |relation|
+              first_record = relation.first
+              yielder.yield(relation, first_record.id)
+            end
+          end
+        end
+
+        private
+
+        def relations_size
+          batch_size = @options[:batch_size] || 1000
+          (@relation.count + batch_size - 1) / batch_size # ceiling division
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/job/iterable/csv_enumerator.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module Job
+    module Iterable
+      # @api private
+      class CsvEnumerator
+        def initialize(csv)
+          unless defined?(CSV) && csv.instance_of?(CSV)
+            raise ArgumentError, "CsvEnumerator.new takes CSV object"
+          end
+
+          @csv = csv
+        end
+
+        def rows(cursor:)
+          @csv.lazy
+            .each_with_index
+            .drop(cursor || 0)
+            .to_enum { count_of_rows_in_file }
+        end
+
+        def batches(cursor:, batch_size: 100)
+          @csv.lazy
+            .each_slice(batch_size)
+            .with_index
+            .drop(cursor || 0)
+            .to_enum { (count_of_rows_in_file.to_f / batch_size).ceil }
+        end
+
+        private
+
+        def count_of_rows_in_file
+          filepath = @csv.path
+          return unless filepath
+
+          count = IO.popen(["wc", "-l", filepath]) do |out|
+            out.read.strip.to_i
+          end
+
+          count -= 1 if @csv.headers
+          count
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/job/iterable/enumerators.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require_relative "active_record_enumerator"
+require_relative "csv_enumerator"
+
+module Sidekiq
+  module Job
+    module Iterable
+      module Enumerators
+        # Builds Enumerator object from a given array, using +cursor+ as an offset.
+        #
+        # @param array [Array]
+        # @param cursor [Integer] offset to start iteration from
+        #
+        # @return [Enumerator]
+        #
+        # @example
+        #   array_enumerator(['build', 'enumerator', 'from', 'any', 'array'], cursor: cursor)
+        #
+        def array_enumerator(array, cursor:)
+          raise ArgumentError, "array must be an Array" unless array.is_a?(Array)
+
+          x = array.each_with_index.drop(cursor || 0)
+          x.to_enum { x.size }
+        end
+
+        # Builds Enumerator from `ActiveRecord::Relation`.
+        # Each Enumerator tick moves the cursor one row forward.
+        #
+        # @param relation [ActiveRecord::Relation] relation to iterate
+        # @param cursor [Object] offset id to start iteration from
+        # @param options [Hash] additional options that will be passed to relevant
+        #   ActiveRecord batching methods
+        #
+        # @return [ActiveRecordEnumerator]
+        #
+        # @example
+        #   def build_enumerator(cursor:)
+        #     active_record_records_enumerator(User.all, cursor: cursor)
+        #   end
+        #
+        #   def each_iteration(user)
+        #     user.notify_about_something
+        #   end
+        #
+        def active_record_records_enumerator(relation, cursor:, **options)
+          ActiveRecordEnumerator.new(relation, cursor: cursor, **options).records
+        end
+
+        # Builds Enumerator from `ActiveRecord::Relation` and enumerates on batches of records.
+        # Each Enumerator tick moves the cursor `:batch_size` rows forward.
+        # @see #active_record_records_enumerator
+        #
+        # @example
+        #   def build_enumerator(product_id, cursor:)
+        #     active_record_batches_enumerator(
+        #       Comment.where(product_id: product_id).select(:id),
+        #       cursor: cursor,
+        #       batch_size: 100
+        #     )
+        #   end
+        #
+        #   def each_iteration(batch_of_comments, product_id)
+        #     comment_ids = batch_of_comments.map(&:id)
+        #     CommentService.call(comment_ids: comment_ids)
+        #   end
+        #
+        def active_record_batches_enumerator(relation, cursor:, **options)
+          ActiveRecordEnumerator.new(relation, cursor: cursor, **options).batches
+        end
+
+        # Builds Enumerator from `ActiveRecord::Relation` and enumerates on batches,
+        # yielding `ActiveRecord::Relation`s.
+        # @see #active_record_records_enumerator
+        #
+        # @example
+        #   def build_enumerator(product_id, cursor:)
+        #     active_record_relations_enumerator(
+        #       Product.find(product_id).comments,
+        #       cursor: cursor,
+        #       batch_size: 100,
+        #     )
+        #   end
+        #
+        #   def each_iteration(batch_of_comments, product_id)
+        #     # batch_of_comments will be a Comment::ActiveRecord_Relation
+        #     batch_of_comments.update_all(deleted: true)
+        #   end
+        #
+        def active_record_relations_enumerator(relation, cursor:, **options)
+          ActiveRecordEnumerator.new(relation, cursor: cursor, **options).relations
+        end
+
+        # Builds Enumerator from a CSV file.
+        #
+        # @param csv [CSV] an instance of CSV object
+        # @param cursor [Integer] offset to start iteration from
+        #
+        # @example
+        #   def build_enumerator(import_id, cursor:)
+        #     import = Import.find(import_id)
+        #     csv_enumerator(import.csv, cursor: cursor)
+        #   end
+        #
+        #   def each_iteration(csv_row)
+        #     # insert csv_row into database
+        #   end
+        #
+        def csv_enumerator(csv, cursor:)
+          CsvEnumerator.new(csv).rows(cursor: cursor)
+        end
+
+        # Builds Enumerator from a CSV file and enumerates on batches of records.
+        #
+        # @param csv [CSV] an instance of CSV object
+        # @param cursor [Integer] offset to start iteration from
+        # @option options :batch_size [Integer] (100) size of the batch
+        #
+        # @example
+        #   def build_enumerator(import_id, cursor:)
+        #     import = Import.find(import_id)
+        #     csv_batches_enumerator(import.csv, cursor: cursor)
+        #   end
+        #
+        #   def each_iteration(batch_of_csv_rows)
+        #     # ...
+        #   end
+        #
+        def csv_batches_enumerator(csv, cursor:, **options)
+          CsvEnumerator.new(csv).batches(cursor: cursor, **options)
+        end
+      end
+    end
+  end
+end