job-iteration 1.3.6 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f1dea81ba8267470285bbd858ace091200060cc0695dfa8a986293139806a4d
-  data.tar.gz: 23affca4d929d7bfafbf9989eb152a65fe3831d380b79b877a31469854768871
+  metadata.gz: 6f875dad03c85d6eb2d3f25d2dd799e6b26082c90e3f70237e7c7a813b513e9d
+  data.tar.gz: 3bd26bbaf398285c98a60c1f6f84daacd47d7ff8f21d10f91bcb37a8fce6071d
 SHA512:
-  metadata.gz: 163fd0b5034c8e0f06fc6f3e8c39f31a8b02286a7d51cc7602ead690cf8696299883c334bf5c57414dc7ba07f48faec157fdc7c602e61cbc2a42b398e23e39aa
-  data.tar.gz: 74634403a068cbccdce59b9ca526cbb97a83dbef2f34dad535385e3d4984d1967dd43748ee32ac0c80fa520ce2a137c33be15e32c2ed0a05e3c7957f85173445
+  metadata.gz: 556835d8b4d7c9d1954936e758fb2c7ecde63063b7260e21e88e5118a603db76dd8689bea28066ea5695c147b2ae7084ed5743450da8b0c62f55463be11c6878
+  data.tar.gz: 05042e53baa957afd99f75d6975958175187dfe77f076d9776c5cf1251b50fd1cb5fda98ddbc39491b60f4cdd6a10036c7f7598dc708cf7ab54a22cb0bfcd796

data/.github/workflows/ci.yml

@@ -6,14 +6,15 @@ jobs:
   build:
     runs-on: ubuntu-latest
     name: Ruby ${{ matrix.ruby }} | Gemfile ${{ matrix.gemfile }}
+    continue-on-error: ${{ matrix.gemfile == 'rails_edge' }}
     services:
       redis:
         image: redis
         ports:
-        - 6379:6379
+          - 6379:6379
     strategy:
       matrix:
-        ruby: ["2.6", "2.7", "3.0", "3.1"]
+        ruby: ["2.6", "2.7", "3.0", "3.1", "3.2"]
         gemfile: [rails_5_2, rails_6_0, rails_6_1, rails_7_0, rails_edge]
         exclude:
           - ruby: "2.6"
@@ -24,31 +25,50 @@ jobs:
             gemfile: rails_5_2
           - ruby: "3.1"
             gemfile: rails_5_2
+          - ruby: "3.2"
+            gemfile: rails_5_2
           - ruby: "3.1"
             gemfile: rails_6_0
+          - ruby: "3.2"
+            gemfile: rails_6_0
+          - ruby: "3.2"
+            gemfile: rails_6_1
+
         include:
           - ruby: head
             gemfile: rails_edge
     env:
       BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
     steps:
-    - name: Check out code
-      uses: actions/checkout@v3
-    - name: Set up Ruby ${{ matrix.ruby }}
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{ matrix.ruby }}
-        bundler-cache: true
-    - name: Start MySQL and create DB
-      run: |
-        sudo systemctl start mysql.service
-        mysql -uroot -h localhost -proot -e "CREATE DATABASE job_iteration_test;"
-    - name: Rubocop
-      run: bundle exec rubocop
-    - name: Ruby tests
-      run: bundle exec rake test
-      env:
-        REDIS_HOST: localhost
-        REDIS_PORT: ${{ job.services.redis.ports[6379] }}
-    - name: Documentation correctly written
-      run: bundle exec yardoc --no-output --no-save --no-stats --fail-on-warning
+      - name: Check out code
+        uses: actions/checkout@v3
+      - name: Set up Ruby ${{ matrix.ruby }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Start MySQL and create DB
+        run: |
+          sudo systemctl start mysql.service
+          mysql -uroot -h localhost -proot -e "CREATE DATABASE job_iteration_test;"
+      - name: Ruby tests
+        run: bundle exec rake test
+        env:
+          REDIS_HOST: localhost
+          REDIS_PORT: ${{ job.services.redis.ports[6379] }}
+
+  lint:
+    runs-on: ubuntu-latest
+    name: Lint
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "3.2"
+          bundler-cache: true
+      - name: Rubocop
+        run: bundle exec rubocop
+      - name: Documentation correctly written
+        run: bundle exec yardoc --no-output --no-save --no-stats --fail-on-warning

data/.github/workflows/cla.yml

@@ -0,0 +1,22 @@
+name: Contributor License Agreement (CLA)
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+  issue_comment:
+    types: [created]
+
+jobs:
+  cla:
+    runs-on: ubuntu-latest
+    if: |
+      (github.event.issue.pull_request 
+        && !github.event.issue.pull_request.merged_at
+        && contains(github.event.comment.body, 'signed')
+      ) 
+      || (github.event.pull_request && !github.event.pull_request.merged)
+    steps:
+      - uses: Shopify/shopify-cla-action@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cla-token: ${{ secrets.CLA_TOKEN }}

data/.rubocop.yml

@@ -2,9 +2,9 @@ inherit_gem:
   rubocop-shopify: rubocop.yml
 
 AllCops:
-  TargetRubyVersion: 2.6.5
+  TargetRubyVersion: 2.6
   Exclude:
-    - 'vendor/bundle/**/*'
+    - "vendor/bundle/**/*"
 Lint/SuppressedException:
   Exclude:
     - lib/job-iteration.rb
@@ -16,5 +16,5 @@ Naming/FileName:
     - lib/job-iteration.rb
 Style/MethodCallWithArgsParentheses:
   Exclude:
-    - 'gemfiles/*'
+    - "gemfiles/*"
     - Gemfile

data/CHANGELOG.md

@@ -1,4 +1,26 @@
-### Master (unreleased)
+### Main (unreleased)
+
+Nil
+
+## v1.4.0 (Aug 23, 2023)
+
+### Changes
+
+- [338](https://github.com/Shopify/job-iteration/pull/338) - All logs are now `ActiveSupport::Notifications` events and logged using `ActiveSupport::LogSubscriber` to allow customization. Events now always include the `cursor_position` tag.
+- [418](https://github.com/Shopify/job-iteration/pull/418) - Return `nil` from `Iteration#perform`, to signal not to rely on return value.
+
+### Features
+
+- [240](https://github.com/Shopify/job-iteration/pull/240) - Allow setting inheritable per-job `job_iteration_max_job_runtime`
+- [310](https://github.com/Shopify/job-iteration/pull/310) - Support nested iteration
+- [341](https://github.com/Shopify/job-iteration/pull/341) - Add `JobIteration.default_retry_backoff`, which sets a default delay when jobs are re-enqueued after being interrupted. Defaults to `nil`, meaning no delay, which matches the current behaviour.
+- [365](https://github.com/Shopify/job-iteration/pull/365) - Support composite primary key as a cursor
+
+### Bug fixes
+
+- [289](https://github.com/Shopify/job-iteration/pull/289) - Fix uninitialized constant error when raising `ConditionNotSupportedError` from `ActiveRecordBatchEnumerator`
+- [346](https://github.com/Shopify/job-iteration/pull/346) - Include failed jobs in `total_time`
+- [417](https://github.com/Shopify/job-iteration/pull/417) - Ensure that numerical values are deserialized as such and not as strings.
 
 ## v1.3.6 (Mar 9, 2022)
 

data/Gemfile

@@ -15,7 +15,6 @@ gem "mysql2", github: "brianmario/mysql2"
 gem "globalid"
 gem "i18n"
 gem "redis"
-gem "database_cleaner"
 
 gem "pry"
 gem "mocha"

data/Gemfile.lock

@@ -1,117 +1,114 @@
 GIT
   remote: https://github.com/brianmario/mysql2
-  revision: 8193dc412c6a266045b9d13a9da36c16750939a4
+  revision: 79f78f940685396f1b5f30ec502544bb7e3ba9cf
   specs:
-    mysql2 (0.5.3)
+    mysql2 (0.5.5)
 
 PATH
   remote: .
   specs:
-    job-iteration (1.3.6)
+    job-iteration (1.4.0)
       activejob (>= 5.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (6.1.4.6)
-      activesupport (= 6.1.4.6)
+    activejob (7.0.7)
+      activesupport (= 7.0.7)
       globalid (>= 0.3.6)
-    activemodel (6.1.4.6)
-      activesupport (= 6.1.4.6)
-    activerecord (6.1.4.6)
-      activemodel (= 6.1.4.6)
-      activesupport (= 6.1.4.6)
-    activesupport (6.1.4.6)
+    activemodel (7.0.7)
+      activesupport (= 7.0.7)
+    activerecord (7.0.7)
+      activemodel (= 7.0.7)
+      activesupport (= 7.0.7)
+    activesupport (7.0.7)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
     ast (2.4.2)
     coderay (1.1.3)
-    concurrent-ruby (1.1.9)
-    connection_pool (2.2.5)
-    database_cleaner (2.0.1)
-      database_cleaner-active_record (~> 2.0.0)
-    database_cleaner-active_record (2.0.1)
-      activerecord (>= 5.a)
-      database_cleaner-core (~> 2.0.0)
-    database_cleaner-core (2.0.1)
-    globalid (1.0.0)
+    concurrent-ruby (1.2.2)
+    connection_pool (2.4.1)
+    globalid (1.1.0)
       activesupport (>= 5.0)
-    i18n (1.10.0)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
+    json (2.6.3)
+    language_server-protocol (3.17.0.3)
     method_source (1.0.0)
-    minitest (5.15.0)
-    mocha (1.13.0)
-    mono_logger (1.1.1)
+    minitest (5.19.0)
+    mocha (2.1.0)
+      ruby2_keywords (>= 0.0.5)
+    mono_logger (1.1.2)
     multi_json (1.15.0)
-    mustermann (1.1.1)
+    mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
-    parallel (1.21.0)
-    parser (3.1.1.0)
+    parallel (1.23.0)
+    parser (3.2.2.3)
       ast (~> 2.4.1)
-    pry (0.14.1)
+      racc
+    pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    rack (2.2.3)
-    rack-protection (2.1.0)
-      rack
+    racc (1.7.1)
+    rack (2.2.8)
+    rack-protection (3.1.0)
+      rack (~> 2.2, >= 2.2.4)
     rainbow (3.1.1)
     rake (13.0.6)
-    redis (4.6.0)
-    redis-namespace (1.8.1)
-      redis (>= 3.0.4)
-    regexp_parser (2.2.1)
-    resque (2.2.0)
+    redis (5.0.7)
+      redis-client (>= 0.9.0)
+    redis-client (0.16.0)
+      connection_pool
+    redis-namespace (1.11.0)
+      redis (>= 4)
+    regexp_parser (2.8.1)
+    resque (2.6.0)
       mono_logger (~> 1.0)
       multi_json (~> 1.0)
       redis-namespace (~> 1.6)
       sinatra (>= 0.9.2)
-      vegas (~> 0.1.2)
     rexml (3.2.5)
-    rubocop (1.25.1)
+    rubocop (1.54.2)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.1.0.0)
+      parser (>= 3.2.2.3)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
-      rexml
-      rubocop-ast (>= 1.15.1, < 2.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.28.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.16.0)
-      parser (>= 3.1.1.0)
-    rubocop-shopify (2.5.0)
-      rubocop (~> 1.25)
-    ruby-progressbar (1.11.0)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.29.0)
+      parser (>= 3.2.1.0)
+    rubocop-shopify (2.14.0)
+      rubocop (~> 1.51)
+    ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
-    sidekiq (6.4.1)
-      connection_pool (>= 2.2.2)
-      rack (~> 2.0)
-      redis (>= 4.2.0)
-    sinatra (2.1.0)
-      mustermann (~> 1.0)
-      rack (~> 2.2)
-      rack-protection (= 2.1.0)
+    sidekiq (7.1.2)
+      concurrent-ruby (< 2)
+      connection_pool (>= 2.3.0)
+      rack (>= 2.2.4)
+      redis-client (>= 0.14.0)
+    sinatra (3.1.0)
+      mustermann (~> 3.0)
+      rack (~> 2.2, >= 2.2.4)
+      rack-protection (= 3.1.0)
       tilt (~> 2.0)
-    sorbet-runtime (0.5.9724)
-    tilt (2.0.10)
-    tzinfo (2.0.4)
+    sorbet-runtime (0.5.10978)
+    tilt (2.2.0)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.1.0)
-    vegas (0.1.11)
-      rack (>= 1.0.0)
-    webrick (1.7.0)
-    yard (0.9.27)
-      webrick (~> 1.7.0)
-    zeitwerk (2.5.4)
+    unicode-display_width (2.4.2)
+    yard (0.9.34)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   activerecord
-  database_cleaner
   globalid
   i18n
   job-iteration!

data/README.md

@@ -69,17 +69,15 @@ class BatchesJob < ApplicationJob
 
   def build_enumerator(product_id, cursor:)
     enumerator_builder.active_record_on_batches(
-      Product.find(product_id).comments,
+      Comment.where(product_id: product_id).select(:id),
       cursor: cursor,
       batch_size: 100,
     )
   end
 
   def each_iteration(batch_of_comments, product_id)
-    # batch_of_comments will contain batches of 100 records
-    batch_of_comments.each do |comment|
-      DeleteCommentJob.perform_later(comment)
-    end
+    comment_ids = batch_of_comments.map(&:id)
+    CommentService.call(comment_ids: comment_ids)
   end
 end
 ```
@@ -126,17 +124,39 @@ class CsvJob < ApplicationJob
     enumerator_builder.csv(import.csv, cursor: cursor)
   end
 
-  def each_iteration(csv_row)
+  def each_iteration(csv_row, import_id)
     # insert csv_row to database
   end
 end
 ```
 
+```ruby
+class NestedIterationJob < ApplicationJob
+  include JobIteration::Iteration
+
+  def build_enumerator(cursor:)
+    enumerator_builder.nested(
+      [
+        ->(cursor) { enumerator_builder.active_record_on_records(Shop.all, cursor: cursor) },
+        ->(shop, cursor) { enumerator_builder.active_record_on_records(shop.products, cursor: cursor) },
+        ->(_shop, product, cursor) { enumerator_builder.active_record_on_batch_relations(product.product_variants, cursor: cursor) }
+      ],
+      cursor: cursor
+    )
+  end
+
+  def each_iteration(product_variants_relation)
+    # do something
+  end
+end
+```
+
 Iteration hooks into Sidekiq and Resque out of the box to support graceful interruption. No extra configuration is required.
 
 ## Guides
 
 * [Iteration: how it works](guides/iteration-how-it-works.md)
+* [Job argument semantics](guides/argument-semantics.md)
 * [Best practices](guides/best-practices.md)
 * [Writing custom enumerator](guides/custom-enumerator.md)
 * [Throttling](guides/throttling.md)
@@ -171,8 +191,6 @@ There a few configuration assumptions that are required for Iteration to work wi
 
 **Why is it important that `each_iteration` takes less than 30 seconds?** When the job worker is scheduled for restart or shutdown, it gets a notice to finish remaining unit of work. To guarantee that no progress is lost we need to make sure that `each_iteration` completes within a reasonable amount of time.
 
-**What do I do if each iteration takes a long time, because it's doing nested operations?** If your `each_iteration` is complex, we recommend enqueuing another job, which will run your nested business logic. We may expose primitives in the future to do this more effectively, but this is not terribly common today.
-
 **Why do I use have to use this ugly helper in `build_enumerator`? Why can't you automatically infer it?** This is how the first version of the API worked. We checked the type of object returned by `build_enumerable`, and whether it was ActiveRecord Relation or an Array, we used the matching adapter. This caused opaque type branching in Iteration internals and it didn’t allow developers to craft their own Enumerators and control the cursor value. We made a decision to _always_ return Enumerator instance from `build_enumerator`. Now we provide explicit helpers to convert ActiveRecord Relation or an Array to Enumerator, and for more complex iteration flows developers can build their own `Enumerator` objects.
 
 **What is the difference between Enumerable and Enumerator?** We recomend [this post](http://blog.arkency.com/2014/01/ruby-to-enum-for-enumerator/) to learn more about Enumerators in Ruby.

data/dev.yml

@@ -7,8 +7,8 @@ up:
     - mysql-client:
         or:        [mysql@5.7]
   - ruby:
-      version: 2.6.5
-  - railgun
+      version: 2.7.6
+  - isogun
   - bundler
   - custom:
       name: Create Job Iteration database

data/guides/argument-semantics.md

@@ -0,0 +1,128 @@
+`job-iteration` overrides the `perform` method of `ActiveJob::Base` to allow for iteration. The `perform` method preserves all the standard calling conventions of the original, but the way the subsequent methods work might differ from what one expects from an ActiveJob subclass.
+
+The call sequence is usually 3 methods:
+
+`perform -> build_enumerator -> each_iteration|each_batch`
+
+In that sense `job-iteration` works like a framework (it calls your code) rather than like a library (that you call). When using jobs with parameters, the following rules of thumb are good to keep in mind.
+
+### Jobs without arguments
+
+Jobs without arguments do not pass anything into either `build_enumerator` or `each_iteration` except for the `cursor` which `job-iteration` persists by itself:
+
+```ruby
+class ArglessJob < ActiveJob::Base
+  include JobIteration::Iteration
+
+  def build_enumerator(cursor:)
+    # ...
+  end
+
+  def each_iteration(single_object_yielded_from_enumerator)
+    # ...
+  end
+end
+```
+
+To enqueue the job:
+
+```ruby
+ArglessJob.perform_later
+```
+
+### Jobs with positional arguments
+
+Jobs with positional arguments will have those arguments available to both `build_enumerator` and `each_iteration`:
+
+```ruby
+class ArgumentativeJob < ActiveJob::Base
+  include JobIteration::Iteration
+
+  def build_enumerator(arg1, arg2, arg3, cursor:)
+    # ...
+  end
+
+  def each_iteration(single_object_yielded_from_enumerator, arg1, arg2, arg3)
+    # ...
+  end
+end
+```
+
+To enqueue the job:
+
+```ruby
+ArgumentativeJob.perform_later(_arg1 = "One", _arg2 = "Two", _arg3 = "Three")
+```
+
+### Jobs with keyword arguments
+
+Jobs with keyword arguments will have the keyword arguments available to both `build_enumerator` and `each_iteration`, but these arguments come packaged into a Hash in both cases. You will need to `fetch` or `[]` your parameter from the `Hash` you get passed in:
+
+```ruby
+class ParameterizedJob < ActiveJob::Base
+  include JobIteration::Iteration
+
+  def build_enumerator(kwargs, cursor:)
+    name = kwargs.fetch(:name)
+    email = kwargs.fetch(:email)
+    # ...
+  end
+
+  def each_iteration(object_yielded_from_enumerator, kwargs)
+    name = kwargs.fetch(:name)
+    email = kwargs.fetch(:email)
+    # ...
+  end
+end
+```
+
+To enqueue the job:
+
+```ruby
+ParameterizedJob.perform_later(name: "Jane", email: "jane@host.example")
+```
+
+Note that you cannot use `ruby2_keywords` at present, and the keyword arguments syntax is not supported in `each_iteration` / `build_enumerator`.
+
+### Jobs with both positional and keyword arguments
+
+Jobs with keyword arguments will have the keyword arguments available to both `build_enumerator` and `each_iteration`, but these arguments come packaged into a Hash in both cases. You will need to `fetch` or `[]` your parameter from the `Hash` you get passed in. Positional arguments get passed first and "unsplatted" (not combined into an array), the `Hash` containing keyword arguments comes after:
+
+```ruby
+class HighlyConfigurableGreetingJob < ActiveJob::Base
+  include JobIteration::Iteration
+
+  def build_enumerator(subject_line, kwargs, cursor:)
+    name = kwargs.fetch(:sender_name)
+    email = kwargs.fetch(:sender_email)
+    # ...
+  end
+
+  def each_iteration(object_yielded_from_enumerator, subject_line, kwargs)
+    name = kwargs.fetch(:sender_name)
+    email = kwargs.fetch(:sender_email)
+    # ...
+  end
+end
+```
+
+To enqueue the job:
+
+```ruby
+HighlyConfigurableGreetingJob.perform_later(_subject_line = "Greetings everybody!", sender_name: "Jane", sender_email: "jane@host.example")
+```
+
+Note that you cannot use `ruby2_keywords` at present, and the keyword arguments syntax is not supported in `each_iteration` / `build_enumerator`.
+
+### Returning (yielding) from enumerators
+
+When defining a custom enumerator (see the [custom enumerator guide](custom-enumerator.md)) you need to yield two positional arguments from it: the object that will be the value for the current iteration (like a single ActiveModel instance, a single number...) and the value you want to be persisted as the `cursor` value should `job-iteration` decide to interrupt you after this iteration. Calling the enumerator with that cursor should return the next object after the one returned in this iteration. That new `cursor` value does not get passed to `each_iteration`:
+
+```ruby
+Enumerator.new do |yielder|
+  # In this case `cursor` is an Integer
+  cursor.upto(99999) do |offset|
+    yielder.yield(fetch_record_at(offset), offset)
+  end
+end
+```