sidekiq-iteration 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 342540da75582c7f102f6ead29643a5196038978c5626b0c6a00a43db04f4f1f
-  data.tar.gz: cfb7cf80031976e5c68b2503d6ab0a13ffe92efc016419ad20944d3c5ce3e56d
+  metadata.gz: 40efca13e06cd7fdcfc1ff59ad08fea8fc731ee1b5560ae5b75b2591379bcb63
+  data.tar.gz: eec2991b40bb67ffc1dcea55f1c0e8acc98a18cd59ad2fd117cd80c1a94c3e79
 SHA512:
-  metadata.gz: 9c22d0b3d74888b394fcb26ca759a0a9a74e762f06c1d655b1dcb94b791c5d4be3880c6a127d8b5c0d149317fa4f25e1a2550026a14bde62b3743de7b058557c
-  data.tar.gz: 13ca6cd11f437d9c1b25e6dbd380f018674b1b80b7b4fece7c3a91c042b59b9dc8f8e1bc6a42a14325ec89ab4af4b022d4c3bd00e15aca6cb6189082e03d099a
+  metadata.gz: 1162ffafc4d157e7a8f9d2b8f69163e90e83431daac129707e16605a9b0df250c1cf2dd9063f651782b01ab0dcd9a0f3848d381981f94ef5a2daf36f43d591be
+  data.tar.gz: 45a1efa4e1e65ae322b7c923cb5795ceda901863afc4d135cb25e1b342c9604e7f90719259b7fd93b8c38b15a5bcb4cab984617ea915fb58055f21936470269c

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 ## master (unreleased)
 
+## 0.3.0 (2023-05-20)
+
+- Allow a default retry backoff to be configured
+
+    ```ruby
+    SidekiqIteration.default_retry_backoff = 10.seconds
+    ```
+
+- Add ability to iterate Active Record enumerators in reverse order
+
+    ```ruby
+    active_record_records_enumerator(User.all, order: :desc)
+    ```
+
 ## 0.2.0 (2022-11-11)
 
 - Fix storing run metadata when the job fails for sidekiq < 6.5.2

data/README.md

@@ -33,7 +33,7 @@ Software that is designed for high availability [must be resilient](https://12fa
 - Ruby 2.7+ (if you need support for older ruby, [open an issue](https://github.com/fatkodima/sidekiq-iteration/issues/new))
 - Sidekiq 6+
 
-## Getting started
+## Installation
 
 Add this line to your application's Gemfile:
 
@@ -45,6 +45,8 @@ And then execute:
 
     $ bundle
 
+## Getting started
+
 In the job, include `SidekiqIteration::Iteration` module and start describing the job with two methods (`build_enumerator` and `each_iteration`) instead of `perform`:
 
 ```ruby
@@ -184,10 +186,10 @@ class CsvJob
 
   def build_enumerator(import_id, cursor:)
     import = Import.find(import_id)
-    csv_enumereator(import.csv, cursor: cursor)
+    csv_enumerator(import.csv, cursor: cursor)
   end
 
-  def each_iteration(csv_row)
+  def each_iteration(csv_row, import_id)
     # insert csv_row to database
   end
 end
@@ -220,6 +222,7 @@ end
 ## Guides
 
 * [Iteration: how it works](guides/iteration-how-it-works.md)
+* [Job argument semantics](guides/argument-semantics.md)
 * [Best practices](guides/best-practices.md)
 * [Writing custom enumerator](guides/custom-enumerator.md)
 * [Throttling](guides/throttling.md)

data/guides/argument-semantics.md

@@ -0,0 +1,130 @@
+# Argument Semantics
+
+`sidekiq-iteration` defines the `perform` method, required by `sidekiq`, to allow for iteration.
+
+The call sequence is usually 3 methods:
+
+`perform -> build_enumerator -> each_iteration`
+
+In that sense `sidekiq-iteration` works like a framework (it calls your code) rather than like a library (that you call). When using jobs with parameters, the following rules of thumb are good to keep in mind.
+
+## Jobs without arguments
+
+Jobs without arguments do not pass anything into either `build_enumerator` or `each_iteration` except for the `cursor` which `sidekiq-iteration` persists by itself:
+
+```ruby
+class ArglessJob
+  include Sidekiq::Job
+  include SidekiqIteration::Iteration
+
+  def build_enumerator(cursor:)
+    # ...
+  end
+
+  def each_iteration(single_object_yielded_from_enumerator)
+    # ...
+  end
+end
+```
+
+To enqueue the job:
+
+```ruby
+ArglessJob.perform_async
+```
+
+## Jobs with positional arguments
+
+Jobs with positional arguments will have those arguments available to both `build_enumerator` and `each_iteration`:
+
+```ruby
+class ArgumentativeJob
+  include Sidekiq::Job
+  include SidekiqIteration::Iteration
+
+  def build_enumerator(arg1, arg2, arg3, cursor:)
+    # ...
+  end
+
+  def each_iteration(single_object_yielded_from_enumerator, arg1, arg2, arg3)
+    # ...
+  end
+end
+```
+
+To enqueue the job:
+
+```ruby
+ArgumentativeJob.perform_async(_arg1 = "One", _arg2 = "Two", _arg3 = "Three")
+```
+
+## Jobs with keyword arguments
+
+Jobs with keyword arguments will have the keyword arguments available to both `build_enumerator` and `each_iteration`, but these arguments come packaged into a Hash in both cases. You will need to `fetch` or `[]` your parameter from the `Hash` you get passed in:
+
+```ruby
+class ParameterizedJob
+  include Sidekiq::Job
+  include SidekiqIteration::Iteration
+
+  def build_enumerator(kwargs, cursor:)
+    name = kwargs.fetch("name")
+    email = kwargs.fetch("email")
+    # ...
+  end
+
+  def each_iteration(object_yielded_from_enumerator, kwargs)
+    name = kwargs.fetch("name")
+    email = kwargs.fetch("email")
+    # ...
+  end
+end
+```
+
+To enqueue the job:
+
+```ruby
+ParameterizedJob.perform_async("name" => "Jane", "email" => "jane@host.example")
+```
+
+## Jobs with both positional and keyword arguments
+
+Jobs with keyword arguments will have the keyword arguments available to both `build_enumerator` and `each_iteration`, but these arguments come packaged into a Hash in both cases. You will need to `fetch` or `[]` your parameter from the `Hash` you get passed in. Positional arguments get passed first and "unsplatted" (not combined into an array), the `Hash` containing keyword arguments comes after:
+
+```ruby
+class HighlyConfigurableGreetingJob
+  include Sidekiq::Job
+  include SidekiqIteration::Iteration
+
+  def build_enumerator(subject_line, kwargs, cursor:)
+    name = kwargs.fetch("sender_name")
+    email = kwargs.fetch("sender_email")
+    # ...
+  end
+
+  def each_iteration(object_yielded_from_enumerator, subject_line, kwargs)
+    name = kwargs.fetch("sender_name")
+    email = kwargs.fetch("sender_email")
+    # ...
+  end
+end
+```
+
+To enqueue the job:
+
+```ruby
+HighlyConfigurableGreetingJob.perform_async(_subject_line = "Greetings everybody!", "sender_name" => "Jane", "sender_email" => "jane@host.example")
+```
+
+## Returning (yielding) from enumerators
+
+When defining a custom enumerator (see the [custom enumerator guide](custom-enumerator.md)) you need to yield two positional arguments from it: the object that will be the value for the current iteration (like a single ActiveModel instance, a single number...) and the value you want to be persisted as the `cursor` value should `sidekiq-iteration` decide to interrupt you after this iteration. Calling the enumerator with that cursor should return the next object after the one returned in this iteration. That new `cursor` value does not get passed to `each_iteration`:
+
+```ruby
+Enumerator.new do |yielder|
+  # In this case `cursor` is an Integer
+  cursor.upto(99999) do |offset|
+    yielder.yield(fetch_record_at(offset), offset)
+  end
+end
+```

data/guides/custom-enumerator.md

@@ -2,6 +2,17 @@
 
 Iteration leverages the [`Enumerator`](https://ruby-doc.org/core-3.1.2/Enumerator.html) pattern from the Ruby standard library, which allows us to use almost any resource as a collection to iterate.
 
+Before writing an enumerator, it is important to understand [how Iteration works](iteration-how-it-works.md) and how
+your enumerator will be used by it. An enumerator must `yield` two things in the following order as positional
+arguments:
+- An object to be processed in a job `each_iteration` method
+- A cursor position, which Iteration will persist if `each_iteration` returns succesfully and the job is forced to shut
+  down. It can be any data type your job backend can serialize and deserialize correctly.
+
+A job that includes Iteration is first started with `nil` as the cursor. When resuming an interrupted job, Iteration
+will deserialize the persisted cursor and pass it to the job's `build_enumerator` method, which your enumerator uses to
+find objects that come _after_ the last successfully processed object.
+
 ## Cursorless Enumerator
 
 Consider a custom Enumerator that takes items from a Redis list. Because a Redis list is essentially a queue, we can ignore the cursor:
@@ -23,7 +34,7 @@ class ListJob
     end
   end
 
-  def each_iteration(item)
+  def each_iteration(item_from_redis)
     # ...
   end
 end
@@ -31,14 +42,15 @@ end
 
 ## Enumerator with cursor
 
-But what about iterating based on a cursor? Consider this Enumerator that wraps third party API (Stripe) for paginated iteration:
+For a more complex example, consider this Enumerator that wraps a third party API (Stripe) for paginated iteration and
+stores a string as the cursor position:
 
 ```ruby
 class StripeListEnumerator
   # @param resource [Stripe::APIResource] The type of Stripe object to request
   # @param params [Hash] Query parameters for the request
   # @param options [Hash] Request options, such as API key or version
-  # @param cursor [String]
+  # @param cursor [nil, String] The Stripe ID of the last item iterated over
   def initialize(resource, params: {}, options: {}, cursor:)
     pagination_params = {}
     pagination_params[:starting_after] = cursor unless cursor.nil?
@@ -59,6 +71,9 @@ class StripeListEnumerator
   def each
     loop do
       @list.each do |item, _index|
+        # The first argument is what gets passed to `each_iteration`.
+        # The second argument (item.id) is going to be persisted as the cursor,
+        # it doesn't get passed to `each_iteration`.
         yield item, item.id
       end
 
@@ -71,26 +86,38 @@ class StripeListEnumerator
 end
 ```
 
+Here we leverage the Stripe cursor pagination where the cursor is an ID of a specific item in the collection. The job
+which uses such an `Enumerator` would then look like so:
+
 ```ruby
-class StripeJob
+class LoadRefundsForChargeJob
   include Sidekiq::Job
   include SidekiqIteration::Iteration
 
-  def build_enumerator(params, cursor:)
+  def build_enumerator(charge_id, cursor:)
     StripeListEnumerator.new(
       Stripe::Refund,
-      params: { charge: "ch_123" },
+      params: { charge: charge_id }, # "charge_id" will be a prefixed Stripe ID such as "chrg_123"
       options: { api_key: "sk_test_123", stripe_version: "2018-01-18" },
       cursor: cursor
     ).to_enumerator
   end
 
-  def each_iteration(stripe_refund, _params)
+  # Note that in this case `each_iteration` will only receive one positional argument per iteration.
+  # If what your enumerator yields is a composite object you will need to unpack it yourself
+  # inside the `each_iteration`.
+  def each_iteration(stripe_refund, charge_id)
     # ...
   end
 end
 ```
 
+and you initiate the job with
+
+```ruby
+LoadRefundsForChargeJob.perform_later(_charge_id = "chrg_345")
+```
+
 ## Notes
 
 We recommend that you read the implementation of the other enumerators that come with the library (`CsvEnumerator`, `ActiveRecordEnumerator`) to gain a better understanding of building Enumerator objects.

data/lib/sidekiq_iteration/active_record_enumerator.rb

@@ -5,11 +5,15 @@ module SidekiqIteration
   class ActiveRecordEnumerator
     SQL_DATETIME_WITH_NSEC = "%Y-%m-%d %H:%M:%S.%6N"
 
-    def initialize(relation, columns: nil, batch_size: 100, cursor: nil)
+    def initialize(relation, columns: nil, batch_size: 100, order: :asc, cursor: nil)
       unless relation.is_a?(ActiveRecord::Relation)
         raise ArgumentError, "relation must be an ActiveRecord::Relation"
       end
 
+      unless order == :asc || order == :desc
+        raise ArgumentError, ":order must be :asc or :desc, got #{order.inspect}"
+      end
+
       @primary_key = "#{relation.table_name}.#{relation.primary_key}"
       @columns = Array(columns&.map(&:to_s) || @primary_key)
       @primary_key_index = @columns.index(@primary_key) || @columns.index(relation.primary_key)
@@ -19,6 +23,7 @@ module SidekiqIteration
                          @columns + [@primary_key]
                        end
       @batch_size = batch_size
+      @order = order
       @cursor = Array.wrap(cursor)
       raise ArgumentError, "Must specify at least one column" if @columns.empty?
       if relation.joins_values.present? && !@columns.all?(/\./)
@@ -31,7 +36,8 @@ module SidekiqIteration
           "You can use other ways to limit the number of rows, e.g. a WHERE condition on the primary key column."
       end
 
-      @base_relation = relation.reorder(@columns.join(", "))
+      ordering = @columns.to_h { |column| [column, @order] }
+      @base_relation = relation.reorder(ordering)
       @iteration_count = 0
     end
 
@@ -152,18 +158,19 @@ module SidekiqIteration
       end
 
       # (x, y) > (a, b) iff (x > a or (x = a and y > b))
+      # (x, y) < (a, b) iff (x < a or (x = a and y < b))
       def build_starts_after_conditions(index, binds)
         column = @columns[index]
 
         if index < @cursor.size - 1
           binds << @cursor[index] << @cursor[index]
-          "#{column} > ? OR (#{column} = ? AND (#{build_starts_after_conditions(index + 1, binds)}))"
+          "#{column} #{@order == :asc ? '>' : '<'} ? OR (#{column} = ? AND (#{build_starts_after_conditions(index + 1, binds)}))"
         else
           binds << @cursor[index]
           if @columns.size == @cursor.size
-            "#{column} > ?"
+            @order == :asc ? "#{column} > ?" : "#{column} < ?"
           else
-            "#{column} >= ?"
+            @order == :asc ? "#{column} >= ?" : "#{column} <= ?"
           end
         end
       end

data/lib/sidekiq_iteration/enumerators.rb

@@ -31,6 +31,7 @@ module SidekiqIteration
     # @option options :columns [Array<String, Symbol>] used to build the actual query for iteration,
     #   defaults to primary key
     # @option options :batch_size [Integer] (100) size of the batch
+    # @option options :order [:asc, :desc] (:asc) specifies iteration order
     #
     # +columns:+ argument is used to build the actual query for iteration. +columns+: defaults to primary key:
     #

data/lib/sidekiq_iteration/iteration.rb

@@ -13,13 +13,13 @@ module SidekiqIteration
       base.extend(Throttling)
 
       base.class_eval do
-        throttle_on(backoff: 0) do |job|
+        throttle_on(backoff: SidekiqIteration.default_retry_backoff) do |job|
           job.class.max_job_runtime &&
             job.start_time &&
             (Time.now.utc - job.start_time) > job.class.max_job_runtime
         end
 
-        throttle_on(backoff: 0) do
+        throttle_on(backoff: SidekiqIteration.default_retry_backoff) do
           defined?(Sidekiq::CLI) &&
             Sidekiq::CLI.instance.launcher.stopping?
         end
@@ -56,16 +56,22 @@ module SidekiqIteration
 
     attr_reader :executions,
       :cursor_position,
-      :start_time,
       :times_interrupted,
-      :total_time,
       :current_run_iterations
 
+    # The time when the job starts running. If the job is interrupted and runs again,
+    # the value is updated.
+    attr_reader :start_time
+
+    # The total time the job has been running, including multiple iterations.
+    # The time isn't reset if the job is interrupted.
+    attr_reader :total_time
+
     # @private
     def initialize
       super
       @arguments = nil
-      @job_iteration_retry_backoff = nil
+      @job_iteration_retry_backoff = SidekiqIteration.default_retry_backoff
       @needs_reenqueue = false
       @current_run_iterations = 0
     end
@@ -191,14 +197,14 @@ module SidekiqIteration
           )
         end
 
-        adjust_total_time
         true
+      ensure
+        adjust_total_time
       end
 
       def reenqueue_iteration_job
         SidekiqIteration.logger.info("[SidekiqIteration::Iteration] Interrupting and re-enqueueing the job cursor_position=#{cursor_position}")
 
-        adjust_total_time
         @times_interrupted += 1
 
         arguments = @arguments

data/lib/sidekiq_iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SidekiqIteration
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/sidekiq_iteration.rb

@@ -22,6 +22,17 @@ module SidekiqIteration
     #
     attr_accessor :max_job_runtime
 
+    # Configures a delay duration to wait before resuming an interrupted job.
+    #
+    # @example
+    #   SidekiqIteration.default_retry_backoff = 10.seconds
+    #
+    # Defaults to nil which means interrupted jobs will be retried immediately.
+    # This value will be ignored when an interruption is raised by a throttle enumerator,
+    # where the throttle backoff value will take precedence over this setting.
+    #
+    attr_accessor :default_retry_backoff
+
     # Set a custom logger for sidekiq-iteration.
     # Defaults to `Sidekiq.logger`.
     #

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-iteration
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - fatkodima
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-11 00:00:00.000000000 Z
+date: 2023-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -35,6 +35,7 @@ files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- guides/argument-semantics.md
 - guides/best-practices.md
 - guides/custom-enumerator.md
 - guides/iteration-how-it-works.md
@@ -71,8 +72,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.4.12
 signing_key:
 specification_version: 4
-summary: Makes your sidekiq jobs interruptible and resumable.
+summary: Makes your long-running sidekiq jobs interruptible and resumable.
 test_files: []