job-iteration 1.4.1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b6129ebd214f775a5a2bad7a91061499a29035eff823fc3cba03fbb32799b0f
-  data.tar.gz: 37f0309527e1e73f9ab755bc7ec8956f0a7eb8373b362f02fa664404b465c148
+  metadata.gz: cdace86b05a5a1d98777e2310d85d75679073ca63b5e52b0e83bd000891198cf
+  data.tar.gz: 4ed5b475931e24bf56d2b8b126bd9354064c438d0ea16f67c10276fb92ccd97b
 SHA512:
-  metadata.gz: 63cab7dba1d5827758bf510168884f56faf1e18fbc50c54ffe187ef714b51c22f0c54b83cfa328f2f517d5c346444ce0f14b34ff5e4ff9a7837cd9f91bf29bd0
-  data.tar.gz: baceac4783910a2cb58e1e500d0b1c8ac20b483c1fe2ce3b57b2c798503451e1ecbe98dede9e0089031d18f83f00dbabee93438a75050b6864b37d67d58b938e
+  metadata.gz: cf9d7a54a8881146a4a0a1750e99dddf8d32c9db3339411fc61c8477a89f5766e9bb99f493257e4484d56e18ef443cb6a2a0300958342774d2e7ebd2b3e23c32
+  data.tar.gz: 966c39a5e89ae26343e07000111ba582724b85b4bf9b7a17f790a10b679e297dca94105d3fb42830f987498fb8982037c80cf3e8130b8bede845f9c7532ade41

data/.github/workflows/ci.yml

@@ -5,8 +5,8 @@ on: [push, pull_request]
 jobs:
   build:
     runs-on: ubuntu-latest
-    name: Ruby ${{ matrix.ruby }} | Gemfile ${{ matrix.gemfile }}
-    continue-on-error: ${{ matrix.gemfile == 'rails_edge' }}
+    name: Ruby ${{ matrix.ruby }} | Rails ${{ matrix.rails }} | Gemfile ${{ matrix.gemfile }}
+    continue-on-error: ${{ matrix.rails == 'edge' }}
     services:
       redis:
         image: redis
@@ -14,34 +14,53 @@ jobs:
           - 6379:6379
     strategy:
       matrix:
-        ruby: ["2.6", "2.7", "3.0", "3.1", "3.2"]
-        gemfile: [rails_5_2, rails_6_0, rails_6_1, rails_7_0, rails_edge]
+        ruby: ["2.6", "2.7", "3.0", "3.1", "3.2", "3.3"]
+        rails: ["5.2", "6.0", "6.1", "7.0", "7.1", "edge"]
+        gemfile: [rails_gems]
         exclude:
           - ruby: "2.6"
-            gemfile: rails_7_0
+            rails: "7.0"
           - ruby: "2.6"
-            gemfile: rails_edge
+            rails: "7.1"
+          - ruby: "2.6"
+            rails: "edge"
+          - ruby: "2.7"
+            rails: "7.1"
+          - ruby: "2.7"
+            rails: "edge"
+          - ruby: "3.0"
+            rails: "5.2"
+          - ruby: "3.0"
+            rails: "7.1"
           - ruby: "3.0"
-            gemfile: rails_5_2
+            rails: "edge"
           - ruby: "3.1"
-            gemfile: rails_5_2
-          - ruby: "3.2"
-            gemfile: rails_5_2
+            rails: "5.2"
           - ruby: "3.1"
-            gemfile: rails_6_0
+            rails: "6.0"
+          - ruby: "3.2"
+            rails: "5.2"
           - ruby: "3.2"
-            gemfile: rails_6_0
+            rails: "6.0"
           - ruby: "3.2"
-            gemfile: rails_6_1
+            rails: "6.1"
+          - ruby: "3.3"
+            rails: "5.2"
+          - ruby: "3.3"
+            rails: "6.0"
+          - ruby: "3.3"
+            rails: "6.1"
 
         include:
           - ruby: head
-            gemfile: rails_edge
+            rails: "edge"
+            gemfile: rails_gems
     env:
       BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
+      RAILS_VERSION: ${{ matrix.rails }}
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Ruby ${{ matrix.ruby }}
         uses: ruby/setup-ruby@v1
         with:
@@ -53,20 +72,16 @@ jobs:
           mysql -uroot -h localhost -proot -e "CREATE DATABASE job_iteration_test;"
       - name: Ruby tests
         run: bundle exec rake test
-        env:
-          REDIS_HOST: localhost
-          REDIS_PORT: ${{ job.services.redis.ports[6379] }}
 
   lint:
     runs-on: ubuntu-latest
     name: Lint
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
-          ruby-version: "3.2"
           bundler-cache: true
       - name: Rubocop
         run: bundle exec rubocop

data/.gitignore

@@ -6,6 +6,6 @@
 /pkg/
 /spec/reports/
 /tmp/
-.ruby-version
 .rubocop-http---shopify-github-io-ruby-style-guide-rubocop-yml
 gemfiles/*.lock
+dump.rdb

data/.rubocop.yml

@@ -1,20 +1,16 @@
 inherit_gem:
   rubocop-shopify: rubocop.yml
 
+inherit_mode:
+  merge:
+    - Include
+
 AllCops:
-  TargetRubyVersion: 2.6
-  Exclude:
-    - "vendor/bundle/**/*"
+  Include:
+    - '**/*.gemfile'
 Lint/SuppressedException:
   Exclude:
     - lib/job-iteration.rb
-Style/GlobalVars:
-  Exclude:
-    - lib/job-iteration/integrations/resque.rb
 Naming/FileName:
   Exclude:
     - lib/job-iteration.rb
-Style/MethodCallWithArgsParentheses:
-  Exclude:
-    - "gemfiles/*"
-    - Gemfile

data/.ruby-version

@@ -0,0 +1 @@
+3.3.0

data/CHANGELOG.md

@@ -1,4 +1,25 @@
 ### Main (unreleased)
+Nil
+
+## v1.5.0 (May 29, 2024)
+### Changes
+
+- [437](https://github.com/Shopify/job-iteration/pull/437) - Use minimum between per-class `job_iteration_max_job_runtime` and `JobIteration.max_job_runtime`, instead of enforcing only setting decreasing values.
+  Because it is possible to change the global or parent values after setting the value on a class, it is not possible to truly enforce the decreasing value constraint. Instead, we now use the minimum between the global value and per-class value. This is considered a non-breaking change, as it should not break any **existing** code, it only removes the constraint on new classes.
+- [443](https://github.com/Shopify/job-iteration/pull/443) - Use Sidekiq `:quit` callback to detect graceful shutdown. This makes job-iteration compatible with Sidekiq run in embedded mode.
+- [445](https://github.com/Shopify/job-iteration/pull/445) - Add the `around_iterate` callback, which runs around each call of `each_iteration`. This adds extensibility to build some generic handlers, such as metrics collection and logging.
+- [450](https://github.com/Shopify/job-iteration/pull/450) - Infer which interruption adapter to use from the queue adapter of the job. This deprecates setting `JobIteration.interruption_adapter = <callable>`, in favor of `JobIteration.register_interruption_adapter(<queue adapter name>, <callable>)`. `JobIteration.interruption_adapter` will be removed in a future release.
+
+### Bug fixes
+
+- [437](https://github.com/Shopify/job-iteration/pull/437) - Defer reading `JobIteration.max_job_runtime` until runtime, instead of closing around the value at the time of job definition.
+- [431](https://github.com/Shopify/job-iteration/pull/431) - Use `#id_value` instead of `send(:id)`
+when generating position for cursor based on `:id` column (Rails 7.1 and above, where composite
+primary models are now supported). This ensures we grab the value of the id column, rather than a
+potentially composite primary key value.
+- [456](https://github.com/Shopify/job-iteration/pull/431) - Use Arel to generate SQL that's type compatible for the
+  cursor pagination conditionals in ActiveRecord cursor. Previously, the cursor would coerce numeric ids to a string value 
+  (e.g.: `... AND id > '1'`)
 
 ## v1.4.1 (Sep 5, 2023)
 

data/Gemfile

@@ -11,6 +11,17 @@ gemspec
 gem "sidekiq"
 gem "resque"
 
+if defined?(@rails_gems_requirements) && @rails_gems_requirements
+  # We avoid the `gem "..."` syntax here so Dependabot doesn't try to update these gems.
+  [
+    "activejob",
+    "activerecord",
+  ].each { |name| gem name, @rails_gems_requirements }
+else
+  # gem "activejob" # Set in gemspec
+  gem "activerecord"
+end
+
 gem "mysql2", github: "brianmario/mysql2"
 gem "globalid"
 gem "i18n"
@@ -22,6 +33,7 @@ gem "mocha"
 gem "rubocop-shopify", require: false
 gem "yard"
 gem "rake"
+gem "csv" # required for Ruby 3.4+
 
 # for unit testing optional sorbet support
 gem "sorbet-runtime"

data/Gemfile.lock

@@ -1,13 +1,13 @@
 GIT
   remote: https://github.com/brianmario/mysql2
-  revision: 79f78f940685396f1b5f30ec502544bb7e3ba9cf
+  revision: 43ea8af635f5e23f054294ef7759320d47f30e5f
   specs:
-    mysql2 (0.5.5)
+    mysql2 (0.5.6)
 
 PATH
   remote: .
   specs:
-    job-iteration (1.4.1)
+    job-iteration (1.5.0)
       activejob (>= 5.2)
 
 GEM
@@ -28,61 +28,58 @@ GEM
       tzinfo (~> 2.0)
     ast (2.4.2)
     coderay (1.1.3)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
+    csv (3.3.0)
     globalid (1.1.0)
       activesupport (>= 5.0)
-    i18n (1.14.1)
+    i18n (1.14.4)
       concurrent-ruby (~> 1.0)
-    json (2.6.3)
+    json (2.7.1)
     language_server-protocol (3.17.0.3)
     method_source (1.0.0)
     minitest (5.19.0)
-    mocha (2.1.0)
+    mocha (2.2.0)
       ruby2_keywords (>= 0.0.5)
     mono_logger (1.1.2)
     multi_json (1.15.0)
-    mustermann (3.0.0)
-      ruby2_keywords (~> 0.0.1)
-    parallel (1.23.0)
-    parser (3.2.2.3)
+    parallel (1.24.0)
+    parser (3.3.0.5)
       ast (~> 2.4.1)
       racc
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    racc (1.7.1)
-    rack (2.2.8)
-    rack-protection (3.1.0)
-      rack (~> 2.2, >= 2.2.4)
+    racc (1.7.3)
+    rack (3.0.9.1)
     rainbow (3.1.1)
-    rake (13.0.6)
-    redis (5.0.7)
-      redis-client (>= 0.9.0)
-    redis-client (0.16.0)
+    rake (13.2.1)
+    redis (5.2.0)
+      redis-client (>= 0.22.0)
+    redis-client (0.22.1)
       connection_pool
     redis-namespace (1.11.0)
       redis (>= 4)
-    regexp_parser (2.8.1)
+    regexp_parser (2.9.0)
     resque (2.6.0)
       mono_logger (~> 1.0)
       multi_json (~> 1.0)
       redis-namespace (~> 1.6)
       sinatra (>= 0.9.2)
-    rexml (3.2.5)
-    rubocop (1.54.2)
+    rexml (3.2.6)
+    rubocop (1.62.1)
       json (~> 2.3)
       language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.2.2.3)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.28.0, < 2.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.29.0)
-      parser (>= 3.2.1.0)
+    rubocop-ast (1.31.2)
+      parser (>= 3.3.0.4)
     rubocop-shopify (2.14.0)
       rubocop (~> 1.51)
     ruby-progressbar (1.13.0)
@@ -92,23 +89,20 @@ GEM
       connection_pool (>= 2.3.0)
       rack (>= 2.2.4)
       redis-client (>= 0.14.0)
-    sinatra (3.1.0)
-      mustermann (~> 3.0)
-      rack (~> 2.2, >= 2.2.4)
-      rack-protection (= 3.1.0)
-      tilt (~> 2.0)
+    sinatra (1.0)
+      rack (>= 1.0)
     sorbet-runtime (0.5.10978)
-    tilt (2.2.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.4.2)
-    yard (0.9.34)
+    unicode-display_width (2.5.0)
+    yard (0.9.36)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   activerecord
+  csv
   globalid
   i18n
   job-iteration!
@@ -124,4 +118,4 @@ DEPENDENCIES
   yard
 
 BUNDLED WITH
-   2.3.5
+   2.5.7

data/README.md

@@ -185,7 +185,7 @@ There a few configuration assumptions that are required for Iteration to work wi
 
 **What happens when my job is interrupted?** A checkpoint will be persisted to Redis after the current `each_iteration`, and the job will be re-enqueued. Once it's popped off the queue, the worker will work off from the next iteration.
 
-**What happens with retries?** An interruption of a job does not count as a retry. The iteration of job that caused the job to fail will be retried and progress will continue from there on.
+**What happens with retries?** An interruption of a job does not count as a retry. If an exception occurs, the job will retry or be discarded as normal using Active Job configuration for the job. If the job retries, it processes the iteration that originally failed and progress will continue from there on if successful.
 
 **What happens if my iteration takes a long time?** We recommend that a single `each_iteration` should take no longer than 30 seconds. In the future, this may raise an exception.
 

data/dev.yml

@@ -3,17 +3,16 @@
 name: job-iteration
 
 up:
-  - homebrew:
-    - mysql-client:
-        or:        [mysql@5.7]
-  - ruby:
-      version: 2.7.6
-  - isogun
+  - packages:
+      - mysql_client
+  - ruby
   - bundler
+  - mysql
+  - redis
   - custom:
       name: Create Job Iteration database
-      meet: mysql -uroot -h job-iteration.railgun -e "CREATE DATABASE job_iteration_test"
-      met?: mysql -uroot -h job-iteration.railgun job_iteration_test -e "SELECT 1" &> /dev/null
+      meet: mysql -uroot -h $MYSQL_HOST -P $MYSQL_PORT -e "CREATE DATABASE job_iteration_test"
+      met?: mysql -uroot -h $MYSQL_HOST -P $MYSQL_PORT job_iteration_test -e "SELECT 1" &> /dev/null
 
 commands:
   test:

data/gemfiles/rails_gems.gemfile

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+rails_version = ENV.fetch("RAILS_VERSION")
+@rails_gems_requirements = case rails_version
+when "edge"         then { github: "rails/rails", branch: "main" }
+when /\A\d+\.\d+\z/ then "~> #{rails_version}.0"
+else                raise "Unsupported RAILS_VERSION: #{rails_version}"
+end
+
+eval_gemfile "../Gemfile"
+
+# https://github.com/rails/rails/pull/44083
+if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.1") &&
+    rails_version != "edge" && Gem::Version.new(rails_version) < Gem::Version.new("7")
+  gem "net-imap", require: false
+  gem "net-pop", require: false
+  gem "net-smtp", require: false
+end

data/guides/custom-enumerator.md

@@ -1,19 +1,25 @@
-Iteration leverages the [Enumerator](https://ruby-doc.org/3.2.1/Enumerator.html) pattern from the Ruby standard library,
+# Custom Enumerator
+
+`Iteration` leverages the [Enumerator](https://ruby-doc.org/3.2.1/Enumerator.html) pattern from the Ruby standard library,
 which allows us to use almost any resource as a collection to iterate.
 
 Before writing an enumerator, it is important to understand [how Iteration works](iteration-how-it-works.md) and how
 your enumerator will be used by it. An enumerator must `yield` two things in the following order as positional
 arguments:
 - An object to be processed in a job `each_iteration` method
-- A cursor position, which Iteration will persist if `each_iteration` returns succesfully and the job is forced to shut
+- A cursor position, which `Iteration` will persist if `each_iteration` returns successfully and the job is forced to shut
   down. It can be any data type your job backend can serialize and deserialize correctly.
 
-A job that includes Iteration is first started with `nil` as the cursor. When resuming an interrupted job, Iteration
+A job that includes `Iteration` is first started with `nil` as the cursor. When resuming an interrupted job, `Iteration`
 will deserialize the persisted cursor and pass it to the job's `build_enumerator` method, which your enumerator uses to
 find objects that come _after_ the last successfully processed object. The [array enumerator](https://github.com/Shopify/job-iteration/blob/v1.3.6/lib/job-iteration/enumerator_builder.rb#L50-L67)
 is a simple example which uses the array index as the cursor position.
 
-For a more complex example, consider this Enumerator that wraps a third party API (Stripe) for paginated iteration and
+In addition to the remainder of this guide, we recommend you read the implementation of the other enumerators that come with the library (`CsvEnumerator`, `ActiveRecordEnumerator`) to gain a better understanding of building enumerators.
+
+## Enumerator with cursor
+
+For a more complex example, consider this `Enumerator` that wraps a third party API (Stripe) for paginated iteration and
 stores a string as the cursor position:
 
 ```ruby
@@ -58,6 +64,8 @@ class StripeListEnumerator
 end
 ```
 
+### Usage
+
 Here we leverage the Stripe cursor pagination where the cursor is an ID of a specific item in the collection. The job
 which uses such an `Enumerator` would then look like so:
 
@@ -90,12 +98,14 @@ end
 and you initiate the job with
 
 ```ruby
-LoadRefundsForChargeJob.perform_later(_charge_id = "chrg_345")
+LoadRefundsForChargeJob.perform_later(charge_id = "chrg_345")
 ```
 
-Sometimes you can ignore the cursor. Consider the following custom Enumerator that takes items from a Redis list, which
+## Cursorless enumerator
+
+Sometimes you can ignore the cursor. Consider the following custom `Enumerator` that takes items from a Redis list, which
 is essentially a queue. Even if this job doesn't need to persist a cursor in order to resume, it can still use
-Iteration's signal handling to finish `each_iteration` and gracefully terminate.
+`Iteration`'s signal handling to finish `each_iteration` and gracefully terminate.
 
 ```ruby
 class RedisPopListJob < ActiveJob::Base
@@ -115,7 +125,9 @@ class RedisPopListJob < ActiveJob::Base
 end
 ```
 
-We recommend that you read the implementation of the other enumerators that come with the library (`CsvEnumerator`, `ActiveRecordEnumerator`) to gain a better understanding of building Enumerator objects.
+## Caveats
+
+### Post-`yield` code
 
 Code that is written after the `yield` in a custom enumerator is not guaranteed to execute. In the case that a job is
 forced to exit ie `job_should_exit?` is true, then the job is re-enqueued during the yield and the rest of the code in

data/guides/iteration-how-it-works.md

@@ -21,6 +21,14 @@ SELECT  `products`.* FROM `products` ORDER BY products.id LIMIT 100
 SELECT  `products`.* FROM `products` WHERE (products.id > 2) ORDER BY products.id LIMIT 100
 ```
 
+## Exceptions inside `each_iteration`
+
+Unrescued exceptions inside the `each_iteration` block are handled the same way as exceptions occuring in `perform` for a regular Active Job subclass, meaning you need to configure it to retry using [`retry_on`](https://api.rubyonrails.org/classes/ActiveJob/Exceptions/ClassMethods.html#method-i-retry_on) or manually call [`retry_job`](https://api.rubyonrails.org/classes/ActiveJob/Exceptions.html#method-i-retry_job). The job will re-enqueue itself with the last successful cursor, the iteration that failed will be retried with the same parameters and the cursor will only move if that iteration succeeds. This behaviour may be enough for intermittent errors, such as network connection failures, but if your execution is deterministic and you have an error, subsequent iterations will never run.
+
+In other words, if you are trying to process 100 records but the job consistently fails on the 61st, only the first 60 will be processed and the job will try to process the 61st record until retries are exhausted.
+
+If no retries are configured or retries are exhausted, Active Job 'bubbles up' the exception to the job backend. Retries by the backend (e.g. Sidekiq) are not supported, meaning that jobs retried by the job backend instead of Active Job will restart from the beginning.
+
 ## Signals
 
 It's critical to know [UNIX signals](https://www.tutorialspoint.com/unix/unix-signals-traps.htm) in order to understand how interruption works. There are two main signals that Sidekiq and Resque use: `SIGTERM` and `SIGKILL`. `SIGTERM` is the graceful termination signal which means that the process should exit _soon_, not immediately. For Iteration, it means that we have time to wait for the last iteration to finish and to push job back to the queue with the last cursor position.

data/isogun.yml

@@ -1,4 +1,3 @@
-# https://development.shopify.io/tools/dev/railgun/Railgun-Config
 name: job-iteration
 
 vm:

data/job-iteration.gemspec

@@ -26,6 +26,5 @@ Gem::Specification.new do |spec|
   spec.metadata["changelog_uri"] = "https://github.com/Shopify/job-iteration/blob/main/CHANGELOG.md"
   spec.metadata["allowed_push_host"] = "https://rubygems.org"
 
-  spec.add_development_dependency("activerecord")
   spec.add_dependency("activejob", ">= 5.2")
 end

data/lib/job-iteration/active_record_cursor.rb

@@ -18,12 +18,8 @@ module JobIteration
       end
     end
 
-    def initialize(relation, columns = nil, position = nil)
-      @columns = if columns
-        Array(columns)
-      else
-        Array(relation.primary_key).map { |pk| "#{relation.table_name}.#{pk}" }
-      end
+    def initialize(relation, columns, position = nil)
+      @columns = columns
       self.position = Array.wrap(position)
       raise ArgumentError, "Must specify at least one column" if columns.empty?
       if relation.joins_values.present? && !@columns.all? { |column| column.to_s.include?(".") }
@@ -34,7 +30,7 @@ module JobIteration
         raise ConditionNotSupportedError
       end
 
-      @base_relation = relation.reorder(@columns.join(","))
+      @base_relation = relation.reorder(*@columns)
       @reached_end = false
     end
 
@@ -54,8 +50,11 @@ module JobIteration
 
     def update_from_record(record)
       self.position = @columns.map do |column|
-        method = column.to_s.split(".").last
-        record.send(method.to_sym)
+        if ActiveRecord.version >= Gem::Version.new("7.1.0.alpha") && column.name == "id"
+          record.id_value
+        else
+          record.send(column.name)
+        end
       end
     end
 
@@ -84,14 +83,14 @@ module JobIteration
       i = @position.size - 1
       column = @columns[i]
       conditions = if @columns.size == @position.size
-        "#{column} > ?"
+        column.gt(@position[i])
       else
-        "#{column} >= ?"
+        column.gteq(@position[i])
       end
       while i > 0
         i -= 1
         column = @columns[i]
-        conditions = "#{column} > ? OR (#{column} = ? AND (#{conditions}))"
+        conditions = column.gt(@position[i]).or(column.eq(@position[i]).and(conditions))
       end
       ret = @position.reduce([conditions]) { |params, value| params << value << value }
       ret.pop

data/lib/job-iteration/active_record_enumerator.rb

@@ -11,9 +11,9 @@ module JobIteration
       @relation = relation
       @batch_size = batch_size
       @columns = if columns
-        Array(columns)
+        Array(columns).map { |col| relation.arel_table[col.to_sym] }
       else
-        Array(relation.primary_key).map { |pk| "#{relation.table_name}.#{pk}" }
+        Array(relation.primary_key).map { |pk| relation.arel_table[pk.to_sym] }
       end
       @cursor = cursor
     end
@@ -45,7 +45,7 @@ module JobIteration
 
     def cursor_value(record)
       positions = @columns.map do |column|
-        attribute_name = column.to_s.split(".").last
+        attribute_name = column.name.to_sym
         column_value(record, attribute_name)
       end
       return positions.first if positions.size == 1
@@ -58,8 +58,8 @@ module JobIteration
     end
 
     def column_value(record, attribute)
-      value = record.read_attribute(attribute.to_sym)
-      case record.class.columns_hash.fetch(attribute).type
+      value = record.read_attribute(attribute)
+      case record.class.columns_hash.fetch(attribute.to_s).type
       when :datetime
         value.strftime(SQL_DATETIME_WITH_NSEC)
       else

data/lib/job-iteration/csv_enumerator.rb

@@ -20,7 +20,7 @@ module JobIteration
     #   csv = CSV.open('tmp/files', { converters: :integer, headers: true })
     #   JobIteration::CsvEnumerator.new(csv).rows(cursor: cursor)
     def initialize(csv)
-      unless csv.instance_of?(CSV)
+      unless defined?(CSV) && csv.instance_of?(CSV)
         raise ArgumentError, "CsvEnumerator.new takes CSV object"
       end
 

data/lib/job-iteration/enumerator_builder.rb

@@ -55,9 +55,6 @@ module JobIteration
       unless enumerable.is_a?(Array)
         raise ArgumentError, "enumerable must be an Array"
       end
-      if enumerable.any? { |i| defined?(ActiveRecord) && i.is_a?(ActiveRecord::Base) }
-        raise ArgumentError, "array cannot contain ActiveRecord objects"
-      end
 
       drop =
         if cursor.nil?

data/lib/job-iteration/interruption_adapters/null_adapter.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# This adapter never interrupts.
+module JobIteration
+  module InterruptionAdapters
+    module NullAdapter
+      class << self
+        def call
+          false
+        end
+      end
+    end
+  end
+end

data/lib/job-iteration/interruption_adapters/resque_adapter.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+begin
+  require "resque"
+rescue LoadError
+  # Resque is not available, no need to load the adapter
+  return
+end
+
+module JobIteration
+  module InterruptionAdapters
+    module ResqueAdapter
+      # @private
+      module IterationExtension
+        def initialize(*)
+          $resque_worker = self # rubocop:disable Style/GlobalVars
+          super
+        end
+      end
+
+      # @private
+      module ::Resque
+        class Worker
+          # The patch is required in order to call shutdown? on a Resque::Worker instance
+          prepend(IterationExtension)
+        end
+      end
+
+      class << self
+        def call
+          $resque_worker.try!(:shutdown?) # rubocop:disable Style/GlobalVars
+        end
+      end
+    end
+
+    register(:resque, ResqueAdapter)
+  end
+end

data/lib/job-iteration/interruption_adapters/sidekiq_adapter.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+begin
+  require "sidekiq"
+rescue LoadError
+  # Sidekiq is not available, no need to load the adapter
+  return
+end
+
+module JobIteration
+  module InterruptionAdapters
+    module SidekiqAdapter
+      class << self
+        attr_accessor :stopping
+
+        def call
+          stopping
+        end
+      end
+
+      ::Sidekiq.configure_server do |config|
+        config.on(:quiet) do
+          SidekiqAdapter.stopping = true
+        end
+      end
+    end
+
+    register(:sidekiq, SidekiqAdapter)
+  end
+end

data/lib/job-iteration/interruption_adapters.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative "interruption_adapters/null_adapter"
+
+module JobIteration
+  module InterruptionAdapters
+    BUNDLED_ADAPTERS = [:resque, :sidekiq].freeze # @api private
+
+    class << self
+      # Returns adapter for specified name.
+      #
+      #   JobIteration::InterruptionAdapters.lookup(:sidekiq)
+      #   # => JobIteration::InterruptionAdapters::SidekiqAdapter
+      def lookup(name)
+        registry.fetch(name.to_sym) do
+          Deprecation.warn(<<~DEPRECATION_MESSAGE, caller_locations(1))
+            No interruption adapter is registered for #{name.inspect}; falling back to `NullAdapter`, which never interrupts.
+            Use `JobIteration::InterruptionAdapters.register(#{name.to_sym.inspect}, <adapter>) to register one.
+            This will raise starting in version #{Deprecation.deprecation_horizon} of #{Deprecation.gem_name}!"
+          DEPRECATION_MESSAGE
+
+          NullAdapter
+        end
+      end
+
+      # Registers adapter for specified name.
+      #
+      #   JobIteration::InterruptionAdapters.register(:sidekiq, MyCustomSidekiqAdapter)
+      def register(name, adapter)
+        raise ArgumentError, "adapter must be callable" unless adapter.respond_to?(:call)
+
+        registry[name.to_sym] = adapter
+      end
+
+      private
+
+      attr_reader :registry
+    end
+
+    @registry = {}
+
+    # Built-in Rails adapters. It doesn't make sense to interrupt for these.
+    register(:async, NullAdapter)
+    register(:inline, NullAdapter)
+    register(:test, NullAdapter)
+
+    # External adapters
+    BUNDLED_ADAPTERS.each do |name|
+      require_relative "interruption_adapters/#{name}_adapter"
+    end
+  end
+end

data/lib/job-iteration/iteration.rb

@@ -42,35 +42,15 @@ module JobIteration
 
     included do |_base|
       define_callbacks :start
+      define_callbacks :iterate
       define_callbacks :shutdown
       define_callbacks :complete
 
       class_attribute(
         :job_iteration_max_job_runtime,
-        instance_writer: false,
+        instance_accessor: false,
         instance_predicate: false,
-        default: JobIteration.max_job_runtime,
       )
-
-      singleton_class.prepend(PrependedClassMethods)
-    end
-
-    module PrependedClassMethods
-      def job_iteration_max_job_runtime=(new)
-        existing = job_iteration_max_job_runtime
-
-        if existing && (!new || new > existing)
-          existing_label = existing.inspect
-          new_label = new ? new.inspect : "#{new.inspect} (no limit)"
-          raise(
-            ArgumentError,
-            "job_iteration_max_job_runtime may only decrease; " \
-              "#{self} tried to increase it from #{existing_label} to #{new_label}",
-          )
-        end
-
-        super
-      end
     end
 
     module ClassMethods
@@ -91,6 +71,10 @@ module JobIteration
         set_callback(:complete, :after, *filters, &blk)
       end
 
+      def around_iterate(&blk)
+        set_callback(:iterate, :around, &blk)
+      end
+
       private
 
       def ban_perform_definition
@@ -136,6 +120,10 @@ module JobIteration
 
     private
 
+    def interruption_adapter # @private
+      JobIteration.interruption_adapter || JobIteration::InterruptionAdapters.lookup(self.class.queue_adapter_name)
+    end
+
     def enumerator_builder
       JobIteration.enumerator_builder.new(self)
     end
@@ -194,7 +182,9 @@ module JobIteration
         tags = instrumentation_tags.merge(cursor_position: cursor_from_enumerator)
         ActiveSupport::Notifications.instrument("each_iteration.iteration", tags) do
           found_record = true
-          each_iteration(object_from_enumerator, *arguments)
+          run_callbacks(:iterate) do
+            each_iteration(object_from_enumerator, *arguments)
+          end
           self.cursor_position = cursor_from_enumerator
         end
 
@@ -291,11 +281,20 @@ module JobIteration
     end
 
     def job_should_exit?
-      if job_iteration_max_job_runtime && start_time && (Time.now.utc - start_time) > job_iteration_max_job_runtime
-        return true
-      end
+      max_job_runtime = job_iteration_max_job_runtime
+      return true if max_job_runtime && start_time && (Time.now.utc - start_time) > max_job_runtime
+
+      interruption_adapter.call || (defined?(super) && super)
+    end
+
+    def job_iteration_max_job_runtime
+      global_max = JobIteration.max_job_runtime
+      class_max = self.class.job_iteration_max_job_runtime
+
+      return global_max unless class_max
+      return class_max unless global_max
 
-      JobIteration.interruption_adapter.call || (defined?(super) && super)
+      [global_max, class_max].min
     end
 
     def handle_completed(completed)

data/lib/job-iteration/railtie.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+return unless defined?(Rails::Railtie)
+
+module JobIteration
+  class Railtie < Rails::Railtie
+    initializer "job_iteration.register_deprecator" do |app|
+      # app.deprecators was added in Rails 7.1
+      app.deprecators[:job_iteration] = JobIteration::Deprecation if app.respond_to?(:deprecators)
+    end
+  end
+end

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.4.1"
+  VERSION = "1.5.0"
 end

data/lib/job-iteration.rb

@@ -3,13 +3,13 @@
 require "active_job"
 require_relative "./job-iteration/version"
 require_relative "./job-iteration/enumerator_builder"
+require_relative "./job-iteration/interruption_adapters"
 require_relative "./job-iteration/iteration"
 require_relative "./job-iteration/log_subscriber"
+require_relative "./job-iteration/railtie"
 
 module JobIteration
-  IntegrationLoadError = Class.new(StandardError)
-
-  INTEGRATIONS = [:resque, :sidekiq]
+  Deprecation = ActiveSupport::Deprecation.new("2.0", "JobIteration")
 
   extend self
 
@@ -29,14 +29,22 @@ module JobIteration
   # This setting will make it to always interrupt a job after it's been iterating for 5 minutes.
   # Defaults to nil which means that jobs will not be interrupted except on termination signal.
   #
-  # This setting can be further reduced (but not increased) by using the inheritable per-class
-  # job_iteration_max_job_runtime setting.
+  # This setting can be overriden by using the inheritable per-class job_iteration_max_job_runtime setting. At runtime,
+  # the lower of the two will be used.
   # @example
   #
   #   class MyJob < ActiveJob::Base
   #     include JobIteration::Iteration
   #     self.job_iteration_max_job_runtime = 1.minute
   #     # ...
+  #
+  # Note that if a sub-class overrides its parent's setting, only the global and sub-class setting will be considered,
+  # not the parent's.
+  # @example
+  #
+  #   class ChildJob < MyJob
+  #     self.job_iteration_max_job_runtime = 3.minutes # MyJob's 1.minute will be discarded.
+  #     # ...
   attr_accessor :max_job_runtime
 
   # Configures a delay duration to wait before resuming an interrupted job.
@@ -49,10 +57,21 @@ module JobIteration
   # where the throttle backoff value will take precedence over this setting.
   attr_accessor :default_retry_backoff
 
-  # Used internally for hooking into job processing frameworks like Sidekiq and Resque.
-  attr_accessor :interruption_adapter
+  attr_reader :interruption_adapter
+
+  # Overrides interruption checks based on queue adapter.
+  # @deprecated - Use JobIteration::InterruptionAdapters.register(:foo, callable) instead.
+  def interruption_adapter=(adapter)
+    Deprecation.warn("Setting JobIteration.interruption_adapter is deprecated. "\
+      "Use JobIteration::InterruptionAdapters.register(:foo, callable) instead "\
+      "to register the callable (a proc, method, or other object responding to #call) "\
+      "as the interruption adapter for queue adapter :foo.")
+    @interruption_adapter = adapter
+  end
 
-  self.interruption_adapter = -> { false }
+  def register_interruption_adapter(adapter_name, adapter)
+    InterruptionAdapters.register(adapter_name, adapter)
+  end
 
   # Set if you want to use your own enumerator builder instead of default EnumeratorBuilder.
   # @example
@@ -65,29 +84,4 @@ module JobIteration
   attr_accessor :enumerator_builder
 
   self.enumerator_builder = JobIteration::EnumeratorBuilder
-
-  def load_integrations
-    loaded = nil
-    INTEGRATIONS.each do |integration|
-      load_integration(integration)
-      if loaded
-        raise IntegrationLoadError,
-          "#{loaded} integration has already been loaded, but #{integration} is also available. " \
-            "Iteration will only work with one integration."
-      end
-      loaded = integration
-    rescue LoadError
-    end
-  end
-
-  def load_integration(integration)
-    unless INTEGRATIONS.include?(integration)
-      raise IntegrationLoadError,
-        "#{integration} integration is not supported. Available integrations: #{INTEGRATIONS.join(", ")}"
-    end
-
-    require_relative "./job-iteration/integrations/#{integration}"
-  end
 end
-
-JobIteration.load_integrations unless ENV["ITERATION_DISABLE_AUTOCONFIGURE"]

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: 1.5.0
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-09-05 00:00:00.000000000 Z
+date: 2024-05-29 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: activerecord
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: activejob
   requirement: !ruby/object:Gem::Requirement
@@ -50,6 +36,7 @@ files:
 - ".github/workflows/cla.yml"
 - ".gitignore"
 - ".rubocop.yml"
+- ".ruby-version"
 - ".yardopts"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -61,11 +48,7 @@ files:
 - bin/setup
 - bin/test
 - dev.yml
-- gemfiles/rails_5_2.gemfile
-- gemfiles/rails_6_0.gemfile
-- gemfiles/rails_6_1.gemfile
-- gemfiles/rails_7_0.gemfile
-- gemfiles/rails_edge.gemfile
+- gemfiles/rails_gems.gemfile
 - guides/argument-semantics.md
 - guides/best-practices.md
 - guides/custom-enumerator.md
@@ -79,11 +62,14 @@ files:
 - lib/job-iteration/active_record_enumerator.rb
 - lib/job-iteration/csv_enumerator.rb
 - lib/job-iteration/enumerator_builder.rb
-- lib/job-iteration/integrations/resque.rb
-- lib/job-iteration/integrations/sidekiq.rb
+- lib/job-iteration/interruption_adapters.rb
+- lib/job-iteration/interruption_adapters/null_adapter.rb
+- lib/job-iteration/interruption_adapters/resque_adapter.rb
+- lib/job-iteration/interruption_adapters/sidekiq_adapter.rb
 - lib/job-iteration/iteration.rb
 - lib/job-iteration/log_subscriber.rb
 - lib/job-iteration/nested_enumerator.rb
+- lib/job-iteration/railtie.rb
 - lib/job-iteration/test_helper.rb
 - lib/job-iteration/throttle_enumerator.rb
 - lib/job-iteration/version.rb
@@ -108,7 +94,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.5.10
 signing_key: 
 specification_version: 4
 summary: Makes your background jobs interruptible and resumable.

data/gemfiles/rails_5_2.gemfile

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-eval_gemfile "../Gemfile"
-
-gem "activejob", "~> 5.2.0"
-gem "activerecord", "~> 5.2.0"

data/gemfiles/rails_6_0.gemfile

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-eval_gemfile "../Gemfile"
-
-gem "activejob", "~> 6.0.0"
-gem "activerecord", "~> 6.0.0"

data/gemfiles/rails_6_1.gemfile

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-eval_gemfile "../Gemfile"
-
-if Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new("3.1")
-  gem "net-imap", require: false
-  gem "net-pop", require: false
-  gem "net-smtp", require: false
-end
-
-gem "activejob", "~> 6.1.0"
-gem "activerecord", "~> 6.1.0"

data/gemfiles/rails_7_0.gemfile

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-eval_gemfile "../Gemfile"
-
-gem "activejob", "~> 7.0.0"
-gem "activerecord", "~> 7.0.0"

data/gemfiles/rails_edge.gemfile

@@ -1,6 +0,0 @@
-# frozen_string_literal: true
-
-eval_gemfile "../Gemfile"
-
-gem "activejob", github: "rails/rails", branch: "main"
-gem "activerecord", github: "rails/rails", branch: "main"

data/lib/job-iteration/integrations/resque.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-require "resque"
-
-module JobIteration
-  module Integrations
-    module ResqueIterationExtension # @private
-      def initialize(*) # @private
-        $resque_worker = self
-        super
-      end
-    end
-
-    # @private
-    module ::Resque
-      class Worker
-        # The patch is required in order to call shutdown? on a Resque::Worker instance
-        prepend(ResqueIterationExtension)
-      end
-    end
-
-    JobIteration.interruption_adapter = -> { $resque_worker.try!(:shutdown?) }
-  end
-end

data/lib/job-iteration/integrations/sidekiq.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-require "sidekiq"
-
-module JobIteration
-  module Integrations # @private
-    JobIteration.interruption_adapter = -> do
-      if defined?(Sidekiq::CLI) && Sidekiq::CLI.instance
-        Sidekiq::CLI.instance.launcher.stopping?
-      else
-        false
-      end
-    end
-  end
-end