job-iteration 1.3.3 → 1.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d715ff6962fc9ff1081d5db93be694b0f0ed3e9fc2935b0d9f252971ea8a40d0
-  data.tar.gz: 60571b6b588816ad637a0eaaee238adb423c230102d4a73745e0fcc3b9e41b17
+  metadata.gz: 2f1dea81ba8267470285bbd858ace091200060cc0695dfa8a986293139806a4d
+  data.tar.gz: 23affca4d929d7bfafbf9989eb152a65fe3831d380b79b877a31469854768871
 SHA512:
-  metadata.gz: 587952d2f628c6d43c982ee8a4fe85d409d4d39044b34aa436bf97a2dd2679d92bfcb098bb2b35380b90297359dbefddc805a188501bd464665e8199a21f673e
-  data.tar.gz: dc172c2fa8172ee47d2af5ed7d47513b8e1131121f6b4f8deebf534a1c7f7e38e8b01f421bf3cc1e419aec9d1c5469020e690757be1e87c35e6d6dc0db8a437c
+  metadata.gz: 163fd0b5034c8e0f06fc6f3e8c39f31a8b02286a7d51cc7602ead690cf8696299883c334bf5c57414dc7ba07f48faec157fdc7c602e61cbc2a42b398e23e39aa
+  data.tar.gz: 74634403a068cbccdce59b9ca526cbb97a83dbef2f34dad535385e3d4984d1967dd43748ee32ac0c80fa520ce2a137c33be15e32c2ed0a05e3c7957f85173445

data/.github/workflows/ci.yml

@@ -1,6 +1,6 @@
 name: CI
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   build:
@@ -13,18 +13,27 @@ jobs:
         - 6379:6379
     strategy:
       matrix:
-        ruby: [2.6, 2.7, 3.0]
-        gemfile: [rails_5_2, rails_6_0, rails_edge]
+        ruby: ["2.6", "2.7", "3.0", "3.1"]
+        gemfile: [rails_5_2, rails_6_0, rails_6_1, rails_7_0, rails_edge]
         exclude:
-          - ruby: 2.6
+          - ruby: "2.6"
+            gemfile: rails_7_0
+          - ruby: "2.6"
             gemfile: rails_edge
-          - ruby: 3.0
+          - ruby: "3.0"
             gemfile: rails_5_2
+          - ruby: "3.1"
+            gemfile: rails_5_2
+          - ruby: "3.1"
+            gemfile: rails_6_0
+        include:
+          - ruby: head
+            gemfile: rails_edge
     env:
       BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
     steps:
     - name: Check out code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Set up Ruby ${{ matrix.ruby }}
       uses: ruby/setup-ruby@v1
       with:

data/CHANGELOG.md

@@ -1,6 +1,18 @@
 ### Master (unreleased)
 
-## v.1.3.3 (Nov 17, 2021)
+## v1.3.6 (Mar 9, 2022)
+
+- [190](https://github.com/Shopify/job-iteration/pull/190) - Fix updating `times_interrupted` and `total_time` when job is throttled
+
+## v1.3.5 (Feb 10, 2022)
+
+- [183](https://github.com/Shopify/job-iteration/pull/183) - Add `JobIteration::EnumeratorBuilder#build_csv_enumerator`
+
+## v1.3.4 (Jan 18, 2022)
+
+- [174](https://github.com/Shopify/job-iteration/pull/174) - Fix Ruby 3.2 compatibility
+
+## v1.3.3 (Nov 17, 2021)
 - [153](https://github.com/Shopify/job-iteration/pull/153) - Re-enqueue jobs only after shutdown hooks have run
 
 ## v1.3.2 (Nov 12, 2021)

data/Gemfile

@@ -11,7 +11,7 @@ gemspec
 gem "sidekiq"
 gem "resque"
 
-gem "mysql2", "~> 0.5"
+gem "mysql2", github: "brianmario/mysql2"
 gem "globalid"
 gem "i18n"
 gem "redis"

data/Gemfile.lock

@@ -1,21 +1,27 @@
+GIT
+  remote: https://github.com/brianmario/mysql2
+  revision: 8193dc412c6a266045b9d13a9da36c16750939a4
+  specs:
+    mysql2 (0.5.3)
+
 PATH
   remote: .
   specs:
-    job-iteration (1.3.3)
+    job-iteration (1.3.6)
       activejob (>= 5.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (6.1.4.1)
-      activesupport (= 6.1.4.1)
+    activejob (6.1.4.6)
+      activesupport (= 6.1.4.6)
       globalid (>= 0.3.6)
-    activemodel (6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activerecord (6.1.4.1)
-      activemodel (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activesupport (6.1.4.1)
+    activemodel (6.1.4.6)
+      activesupport (= 6.1.4.6)
+    activerecord (6.1.4.6)
+      activemodel (= 6.1.4.6)
+      activesupport (= 6.1.4.6)
+    activesupport (6.1.4.6)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -31,20 +37,19 @@ GEM
       activerecord (>= 5.a)
       database_cleaner-core (~> 2.0.0)
     database_cleaner-core (2.0.1)
-    globalid (0.5.2)
+    globalid (1.0.0)
       activesupport (>= 5.0)
-    i18n (1.8.11)
+    i18n (1.10.0)
       concurrent-ruby (~> 1.0)
     method_source (1.0.0)
-    minitest (5.14.4)
+    minitest (5.15.0)
     mocha (1.13.0)
     mono_logger (1.1.1)
     multi_json (1.15.0)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
-    mysql2 (0.5.3)
     parallel (1.21.0)
-    parser (3.0.2.0)
+    parser (3.1.1.0)
       ast (~> 2.4.1)
     pry (0.14.1)
       coderay (~> 1.1)
@@ -52,12 +57,12 @@ GEM
     rack (2.2.3)
     rack-protection (2.1.0)
       rack
-    rainbow (3.0.0)
+    rainbow (3.1.1)
     rake (13.0.6)
-    redis (4.5.1)
+    redis (4.6.0)
     redis-namespace (1.8.1)
       redis (>= 3.0.4)
-    regexp_parser (2.1.1)
+    regexp_parser (2.2.1)
     resque (2.2.0)
       mono_logger (~> 1.0)
       multi_json (~> 1.0)
@@ -65,22 +70,22 @@ GEM
       sinatra (>= 0.9.2)
       vegas (~> 0.1.2)
     rexml (3.2.5)
-    rubocop (1.22.1)
+    rubocop (1.25.1)
       parallel (~> 1.10)
-      parser (>= 3.0.0.0)
+      parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.12.0, < 2.0)
+      rubocop-ast (>= 1.15.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.12.0)
-      parser (>= 3.0.1.1)
-    rubocop-shopify (2.3.0)
-      rubocop (~> 1.22)
+    rubocop-ast (1.16.0)
+      parser (>= 3.1.1.0)
+    rubocop-shopify (2.5.0)
+      rubocop (~> 1.25)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.5)
-    sidekiq (6.3.1)
+    sidekiq (6.4.1)
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
       redis (>= 4.2.0)
@@ -89,15 +94,17 @@ GEM
       rack (~> 2.2)
       rack-protection (= 2.1.0)
       tilt (~> 2.0)
-    sorbet-runtime (0.5.9219)
+    sorbet-runtime (0.5.9724)
     tilt (2.0.10)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.1.0)
     vegas (0.1.11)
       rack (>= 1.0.0)
-    yard (0.9.26)
-    zeitwerk (2.4.2)
+    webrick (1.7.0)
+    yard (0.9.27)
+      webrick (~> 1.7.0)
+    zeitwerk (2.5.4)
 
 PLATFORMS
   ruby
@@ -109,7 +116,7 @@ DEPENDENCIES
   i18n
   job-iteration!
   mocha
-  mysql2 (~> 0.5)
+  mysql2!
   pry
   rake
   redis
@@ -120,4 +127,4 @@ DEPENDENCIES
   yard
 
 BUNDLED WITH
-   2.2.22
+   2.3.5

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018 Shopify
+Copyright (c) 2018-present Shopify
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -2,7 +2,7 @@
 
 [![CI](https://github.com/Shopify/job-iteration/actions/workflows/ci.yml/badge.svg)](https://github.com/Shopify/job-iteration/actions/workflows/ci.yml)
 
-Meet Iteration, an extension for [ActiveJob](https://github.com/rails/rails/tree/master/activejob) that makes your jobs interruptible and resumable, saving all progress that the job has made (aka checkpoint for jobs).
+Meet Iteration, an extension for [ActiveJob](https://github.com/rails/rails/tree/main/activejob) that makes your jobs interruptible and resumable, saving all progress that the job has made (aka checkpoint for jobs).
 
 ## Background
 
@@ -123,7 +123,7 @@ class CsvJob < ApplicationJob
 
   def build_enumerator(import_id, cursor:)
     import = Import.find(import_id)
-    JobIteration::CsvEnumerator.new(import.csv).rows(cursor: cursor)
+    enumerator_builder.csv(import.csv, cursor: cursor)
   end
 
   def each_iteration(csv_row)
@@ -223,4 +223,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Job::Iteration project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/Shopify/job-iteration/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Job::Iteration project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/Shopify/job-iteration/blob/main/CODE_OF_CONDUCT.md).

data/gemfiles/rails_6_1.gemfile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+eval_gemfile "../Gemfile"
+
+if Gem::Version.new(RUBY_VERSION.dup) >= Gem::Version.new("3.1")
+  gem "net-imap", require: false
+  gem "net-pop", require: false
+  gem "net-smtp", require: false
+end
+
+gem "activejob", "~> 6.1.0"
+gem "activerecord", "~> 6.1.0"

data/gemfiles/rails_7_0.gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+eval_gemfile "../Gemfile"
+
+gem "activejob", "~> 7.0.0"
+gem "activerecord", "~> 7.0.0"

data/guides/best-practices.md

@@ -41,6 +41,14 @@ ActiveSupport::Notifications.subscribe('interrupted.iteration') do |_, _, _, _,
     tags: { job_class: tags[:job_class]&.underscore }
   )
 end
+
+# If you're using ThrottleEnumerator
+ActiveSupport::Notifications.subscribe('throttled.iteration') do |_, _, _, _, tags|
+  StatsD.increment(
+    "iteration.throttled",
+    tags: { job_class: tags[:job_class]&.underscore }
+  )
+end
 ```
 
 ## Max iteration time

data/job-iteration.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.metadata["changelog_uri"] = "https://github.com/Shopify/job-iteration/blob/master/CHANGELOG.md"
+  spec.metadata["changelog_uri"] = "https://github.com/Shopify/job-iteration/blob/main/CHANGELOG.md"
   spec.metadata["allowed_push_host"] = "https://rubygems.org"
 
   spec.add_development_dependency("activerecord")

data/lib/job-iteration/active_record_batch_enumerator.rb

@@ -34,6 +34,7 @@ module JobIteration
 
     def each
       return to_enum { size } unless block_given?
+
       while (relation = next_batch)
         yield relation, cursor_value
       end
@@ -65,7 +66,8 @@ module JobIteration
       @cursor = Array.wrap(cursor)
 
       # Yields relations by selecting the primary keys of records in the batch.
-      # Post.where(published: nil) results in an enumerator of relations like: Post.where(ids: batch_of_ids)
+      # Post.where(published: nil) results in an enumerator of relations like:
+      # Post.where(published: nil, ids: batch_of_ids)
       @base_relation.where(@primary_key => ids)
     end
 
@@ -85,6 +87,7 @@ module JobIteration
 
     def cursor_value
       return @cursor.first if @cursor.size == 1
+
       @cursor
     end
 

data/lib/job-iteration/active_record_cursor.rb

@@ -45,6 +45,7 @@ module JobIteration
 
     def position=(position)
       raise "Cursor position cannot contain nil values" if position.any?(&:nil?)
+
       @position = position
     end
 

data/lib/job-iteration/active_record_enumerator.rb

@@ -45,6 +45,7 @@ module JobIteration
         column_value(record, attribute_name)
       end
       return positions.first if positions.size == 1
+
       positions
     end
 

data/lib/job-iteration/enumerator_builder.rb

@@ -43,6 +43,7 @@ module JobIteration
     # Builds Enumerator objects that iterates N times and yields number starting from zero.
     def build_times_enumerator(number, cursor:)
       raise ArgumentError, "First argument must be an Integer" unless number.is_a?(Integer)
+
       wrap(self, build_array_enumerator(number.times.to_a, cursor: cursor))
     end
 
@@ -54,6 +55,7 @@ module JobIteration
       if enumerable.any? { |i| defined?(ActiveRecord) && i.is_a?(ActiveRecord::Base) }
         raise ArgumentError, "array cannot contain ActiveRecord objects"
       end
+
       drop =
         if cursor.nil?
           0
@@ -119,12 +121,14 @@ module JobIteration
 
     # Builds Enumerator from Active Record Relation and enumerates on batches, yielding Active Record Relations.
     # See documentation for #build_active_record_enumerator_on_batches.
-    def build_active_record_enumerator_on_batch_relations(scope, cursor:, **args)
-      JobIteration::ActiveRecordBatchEnumerator.new(
+    def build_active_record_enumerator_on_batch_relations(scope, wrap: true, cursor:, **args)
+      enum = JobIteration::ActiveRecordBatchEnumerator.new(
         scope,
         cursor: cursor,
         **args
       ).each
+      enum = wrap(self, enum) if wrap
+      enum
     end
 
     def build_throttle_enumerator(enum, throttle_on:, backoff:)
@@ -136,6 +140,10 @@ module JobIteration
       ).to_enum
     end
 
+    def build_csv_enumerator(enumerable, cursor:)
+      CsvEnumerator.new(enumerable).rows(cursor: cursor)
+    end
+
     alias_method :once, :build_once_enumerator
     alias_method :times, :build_times_enumerator
     alias_method :array, :build_array_enumerator
@@ -143,6 +151,7 @@ module JobIteration
     alias_method :active_record_on_batches, :build_active_record_enumerator_on_batches
     alias_method :active_record_on_batch_relations, :build_active_record_enumerator_on_batch_relations
     alias_method :throttle, :build_throttle_enumerator
+    alias_method :csv, :build_csv_enumerator
 
     private
 

data/lib/job-iteration/integrations/resque.rb

@@ -10,8 +10,14 @@ module JobIteration
         super
       end
     end
-    # The patch is required in order to call shutdown? on a Resque::Worker instance
-    Resque::Worker.prepend(ResqueIterationExtension)
+
+    # @private
+    module ::Resque
+      class Worker
+        # The patch is required in order to call shutdown? on a Resque::Worker instance
+        prepend(ResqueIterationExtension)
+      end
+    end
 
     JobIteration.interruption_adapter = -> { $resque_worker.try!(:shutdown?) }
   end

data/lib/job-iteration/iteration.rb

@@ -68,6 +68,7 @@ module JobIteration
 
     def initialize(*arguments)
       super
+      @job_iteration_retry_backoff = nil
       @needs_reenqueue = false
       self.times_interrupted = 0
       self.total_time = 0.0
@@ -132,10 +133,11 @@ module JobIteration
       end
 
       run_callbacks(:shutdown)
+      completed = handle_completed(completed)
 
       if @needs_reenqueue
         reenqueue_iteration_job
-      elsif run_complete_callbacks?(completed)
+      elsif completed
         run_callbacks(:complete)
         output_interrupt_summary
       end
@@ -157,6 +159,7 @@ module JobIteration
         end
 
         next unless job_should_exit?
+
         self.executions -= 1 if executions > 1
         @needs_reenqueue = true
         return false
@@ -184,7 +187,7 @@ module JobIteration
       self.times_interrupted += 1
 
       self.already_in_queue = true if respond_to?(:already_in_queue=)
-      retry_job
+      retry_job(wait: @job_iteration_retry_backoff)
     end
 
     def adjust_total_time
@@ -266,24 +269,29 @@ module JobIteration
       JobIteration.interruption_adapter.call || (defined?(super) && super)
     end
 
-    def run_complete_callbacks?(completed)
-      # nil means that someone aborted the job but want to call the on_complete callback
-      if completed.nil?
-        completed = :finished
-      end
-
+    def handle_completed(completed)
       case completed
-      when :finished, true then true
-      # skip_complete_callbacks is returning from ThrottleEnumeratorand we do not want the on_complete callback to
-      # be executed
-      when false, :skip_complete_callbacks then false
+      when nil # someone aborted the job but wants to call the on_complete callback
+        return true
+      when true
+        return true
+      when false, :skip_complete_callbacks
+        return false
+      when Array # used by ThrottleEnumerator
+        reason, backoff = completed
+        raise "Unknown reason: #{reason}" unless reason == :retry
+
+        @job_iteration_retry_backoff = backoff
+        @needs_reenqueue = true
+        return false
       end
+      raise "Unexpected thrown value: #{completed.inspect}"
     end
 
     def valid_cursor_parameter?(parameters)
       # this condition is when people use the splat operator.
       # def build_enumerator(*)
-      return true if parameters == [[:rest]]
+      return true if parameters == [[:rest]] || parameters == [[:rest, :*]]
 
       parameters.each do |parameter_type, parameter_name|
         next unless parameter_name == :cursor

data/lib/job-iteration/throttle_enumerator.rb

@@ -31,8 +31,7 @@ module JobIteration
         @enum.each do |*val|
           if should_throttle?
             ActiveSupport::Notifications.instrument("throttled.iteration", job_class: @job.class.name)
-            @job.retry_job(wait: @backoff)
-            throw(:abort, :skip_complete_callbacks)
+            throw(:abort, [:retry, @backoff])
           end
 
           yielder.yield(*val)

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.3.3"
+  VERSION = "1.3.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.3.3
+  version: 1.3.6
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-11-17 00:00:00.000000000 Z
+date: 2022-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -62,6 +62,8 @@ files:
 - dev.yml
 - gemfiles/rails_5_2.gemfile
 - gemfiles/rails_6_0.gemfile
+- gemfiles/rails_6_1.gemfile
+- gemfiles/rails_7_0.gemfile
 - gemfiles/rails_edge.gemfile
 - guides/best-practices.md
 - guides/custom-enumerator.md
@@ -85,7 +87,7 @@ homepage: https://github.com/shopify/job-iteration
 licenses:
 - MIT
 metadata:
-  changelog_uri: https://github.com/Shopify/job-iteration/blob/master/CHANGELOG.md
+  changelog_uri: https://github.com/Shopify/job-iteration/blob/main/CHANGELOG.md
   allowed_push_host: https://rubygems.org
 post_install_message: 
 rdoc_options: []