job-iteration 1.3.1 → 1.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8788b6f2da0dba53c31c57d529a371e3e822091caa3654bcafc6cf25ad031adb
-  data.tar.gz: 14cba8e6969be4d6aba0f729abd355196324f2eebd00d9b659053b7efe25117d
+  metadata.gz: 2d6eb941d1755c4ddb2e249e5aa50fdef6719ca173ec59643b71b4c1f724ea67
+  data.tar.gz: 3f75f0be9e4023c3a1f2dce9c63f311d85a72fb599b12d6b0196b3546a4db092
 SHA512:
-  metadata.gz: 97f3d383f9eb1e82c3c68ee21bdcaf31a5df2923ebd4eb9089b7b78348f64bd69aa0da39acd0eba23cd309263931639e4e6ca702511a0d83a9528e520ff77e89
-  data.tar.gz: beda68f91cfd51d0d78c2b0c26b20b70bc339fc25591d3ceefe825e687bdc057b448832d2abead1e80c5259eda52ef04d6b6cf83063431766ab8920dd753c396
+  metadata.gz: 9552d7e14685aa4aa9f69456bbc51e57d685903b2fc0752bfd2bf01f6140aa19d375000ae3c11a2f4cef8a78a5a3e87934cc542c2f719698ab539f00e6cadadd
+  data.tar.gz: 9fc95b241595dd1268981cd8156f8c587b71dad5cbcd7eb931fe3ee0ea723ba23521c38ddcf0658f4b4d413ed46b9c885e2dd45f2155c6477129869575390f1f

data/.github/workflows/ci.yml

@@ -13,13 +13,18 @@ jobs:
         - 6379:6379
     strategy:
       matrix:
-        ruby: [2.6, 2.7, 3.0]
+        ruby: ["2.6", "2.7", "3.0"]
         gemfile: [rails_5_2, rails_6_0, rails_edge]
         exclude:
-          - ruby: 2.6
+          - ruby: "2.6"
             gemfile: rails_edge
-          - ruby: 3.0
+          - ruby: "3.0"
             gemfile: rails_5_2
+        include:
+          - ruby: "3.1"
+            gemfile: rails_edge
+          - ruby: head
+            gemfile: rails_edge
     env:
       BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
     steps:

data/CHANGELOG.md

@@ -1,7 +1,21 @@
 ### Master (unreleased)
 
+## v.1.3.5 (Feb 10, 2022)
+
+- [183](https://github.com/Shopify/job-iteration/pull/183) - Add `JobIteration::EnumeratorBuilder#build_csv_enumerator`
+
+## v.1.3.4 (Jan 18, 2022)
+
+- [174](https://github.com/Shopify/job-iteration/pull/174) - Fix Ruby 3.2 compatibility
+
+## v.1.3.3 (Nov 17, 2021)
+- [153](https://github.com/Shopify/job-iteration/pull/153) - Re-enqueue jobs only after shutdown hooks have run
+
+## v1.3.2 (Nov 12, 2021)
+- [148](https://github.com/Shopify/job-iteration/pull/148) - Revert "Do not evaluate enumerator when throttled", due to backwards incompatibility.
+
 ## v1.3.1 (Nov 11, 2021)
-- [87](https://github.com/Shopify/job-iteration/pull/87) - Do not evaluate enumerator when throttled
+- [87](https://github.com/Shopify/job-iteration/pull/87) - Do not evaluate enumerator when throttled (REVERTED)
 
 
 ## v1.3.0 (Oct 7, 2021)

data/Gemfile

@@ -11,7 +11,7 @@ gemspec
 gem "sidekiq"
 gem "resque"
 
-gem "mysql2", "~> 0.5"
+gem "mysql2", github: "brianmario/mysql2"
 gem "globalid"
 gem "i18n"
 gem "redis"

data/Gemfile.lock

@@ -1,21 +1,27 @@
+GIT
+  remote: https://github.com/brianmario/mysql2
+  revision: 8193dc412c6a266045b9d13a9da36c16750939a4
+  specs:
+    mysql2 (0.5.3)
+
 PATH
   remote: .
   specs:
-    job-iteration (1.3.1)
+    job-iteration (1.3.5)
       activejob (>= 5.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (6.1.4.1)
-      activesupport (= 6.1.4.1)
+    activejob (6.1.4.4)
+      activesupport (= 6.1.4.4)
       globalid (>= 0.3.6)
-    activemodel (6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activerecord (6.1.4.1)
-      activemodel (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activesupport (6.1.4.1)
+    activemodel (6.1.4.4)
+      activesupport (= 6.1.4.4)
+    activerecord (6.1.4.4)
+      activemodel (= 6.1.4.4)
+      activesupport (= 6.1.4.4)
+    activesupport (6.1.4.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -31,20 +37,19 @@ GEM
       activerecord (>= 5.a)
       database_cleaner-core (~> 2.0.0)
     database_cleaner-core (2.0.1)
-    globalid (0.5.2)
+    globalid (1.0.0)
       activesupport (>= 5.0)
-    i18n (1.8.10)
+    i18n (1.9.1)
       concurrent-ruby (~> 1.0)
     method_source (1.0.0)
-    minitest (5.14.4)
+    minitest (5.15.0)
     mocha (1.13.0)
     mono_logger (1.1.1)
     multi_json (1.15.0)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
-    mysql2 (0.5.3)
     parallel (1.21.0)
-    parser (3.0.2.0)
+    parser (3.1.0.0)
       ast (~> 2.4.1)
     pry (0.14.1)
       coderay (~> 1.1)
@@ -52,35 +57,35 @@ GEM
     rack (2.2.3)
     rack-protection (2.1.0)
       rack
-    rainbow (3.0.0)
+    rainbow (3.1.1)
     rake (13.0.6)
-    redis (4.5.1)
+    redis (4.6.0)
     redis-namespace (1.8.1)
       redis (>= 3.0.4)
-    regexp_parser (2.1.1)
-    resque (2.1.0)
+    regexp_parser (2.2.0)
+    resque (2.2.0)
       mono_logger (~> 1.0)
       multi_json (~> 1.0)
       redis-namespace (~> 1.6)
       sinatra (>= 0.9.2)
       vegas (~> 0.1.2)
     rexml (3.2.5)
-    rubocop (1.22.1)
+    rubocop (1.24.1)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.12.0, < 2.0)
+      rubocop-ast (>= 1.15.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.12.0)
+    rubocop-ast (1.15.1)
       parser (>= 3.0.1.1)
-    rubocop-shopify (2.3.0)
-      rubocop (~> 1.22)
+    rubocop-shopify (2.4.0)
+      rubocop (~> 1.24)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.5)
-    sidekiq (6.2.2)
+    sidekiq (6.4.0)
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
       redis (>= 4.2.0)
@@ -89,7 +94,7 @@ GEM
       rack (~> 2.2)
       rack-protection (= 2.1.0)
       tilt (~> 2.0)
-    sorbet-runtime (0.5.9219)
+    sorbet-runtime (0.5.9619)
     tilt (2.0.10)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
@@ -97,7 +102,7 @@ GEM
     vegas (0.1.11)
       rack (>= 1.0.0)
     yard (0.9.26)
-    zeitwerk (2.4.2)
+    zeitwerk (2.5.1)
 
 PLATFORMS
   ruby
@@ -109,7 +114,7 @@ DEPENDENCIES
   i18n
   job-iteration!
   mocha
-  mysql2 (~> 0.5)
+  mysql2!
   pry
   rake
   redis
@@ -120,4 +125,4 @@ DEPENDENCIES
   yard
 
 BUNDLED WITH
-   2.2.20
+   2.3.5

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018 Shopify
+Copyright (c) 2018-present Shopify
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -2,7 +2,7 @@
 
 [![CI](https://github.com/Shopify/job-iteration/actions/workflows/ci.yml/badge.svg)](https://github.com/Shopify/job-iteration/actions/workflows/ci.yml)
 
-Meet Iteration, an extension for [ActiveJob](https://github.com/rails/rails/tree/master/activejob) that makes your jobs interruptible and resumable, saving all progress that the job has made (aka checkpoint for jobs).
+Meet Iteration, an extension for [ActiveJob](https://github.com/rails/rails/tree/main/activejob) that makes your jobs interruptible and resumable, saving all progress that the job has made (aka checkpoint for jobs).
 
 ## Background
 
@@ -123,7 +123,7 @@ class CsvJob < ApplicationJob
 
   def build_enumerator(import_id, cursor:)
     import = Import.find(import_id)
-    JobIteration::CsvEnumerator.new(import.csv).rows(cursor: cursor)
+    enumerator_builder.csv(import.csv, cursor: cursor)
   end
 
   def each_iteration(csv_row)
@@ -223,4 +223,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Job::Iteration project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/Shopify/job-iteration/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Job::Iteration project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/Shopify/job-iteration/blob/main/CODE_OF_CONDUCT.md).

data/guides/best-practices.md

@@ -41,6 +41,14 @@ ActiveSupport::Notifications.subscribe('interrupted.iteration') do |_, _, _, _,
     tags: { job_class: tags[:job_class]&.underscore }
   )
 end
+
+# If you're using ThrottleEnumerator
+ActiveSupport::Notifications.subscribe('throttled.iteration') do |_, _, _, _, tags|
+  StatsD.increment(
+    "iteration.throttled",
+    tags: { job_class: tags[:job_class]&.underscore }
+  )
+end
 ```
 
 ## Max iteration time

data/job-iteration.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.metadata["changelog_uri"] = "https://github.com/Shopify/job-iteration/blob/master/CHANGELOG.md"
+  spec.metadata["changelog_uri"] = "https://github.com/Shopify/job-iteration/blob/main/CHANGELOG.md"
   spec.metadata["allowed_push_host"] = "https://rubygems.org"
 
   spec.add_development_dependency("activerecord")

data/lib/job-iteration/active_record_batch_enumerator.rb

@@ -40,7 +40,7 @@ module JobIteration
     end
 
     def size
-      (@base_relation.count + @batch_size - 1) / @batch_size # ceiling division
+      (@base_relation.count(:all) + @batch_size - 1) / @batch_size # ceiling division
     end
 
     private
@@ -65,7 +65,8 @@ module JobIteration
       @cursor = Array.wrap(cursor)
 
       # Yields relations by selecting the primary keys of records in the batch.
-      # Post.where(published: nil) results in an enumerator of relations like: Post.where(ids: batch_of_ids)
+      # Post.where(published: nil) results in an enumerator of relations like:
+      # Post.where(published: nil, ids: batch_of_ids)
       @base_relation.where(@primary_key => ids)
     end
 

data/lib/job-iteration/active_record_enumerator.rb

@@ -34,7 +34,7 @@ module JobIteration
     end
 
     def size
-      @relation.count
+      @relation.count(:all)
     end
 
     private

data/lib/job-iteration/enumerator_builder.rb

@@ -119,12 +119,14 @@ module JobIteration
 
     # Builds Enumerator from Active Record Relation and enumerates on batches, yielding Active Record Relations.
     # See documentation for #build_active_record_enumerator_on_batches.
-    def build_active_record_enumerator_on_batch_relations(scope, cursor:, **args)
-      JobIteration::ActiveRecordBatchEnumerator.new(
+    def build_active_record_enumerator_on_batch_relations(scope, wrap: true, cursor:, **args)
+      enum = JobIteration::ActiveRecordBatchEnumerator.new(
         scope,
         cursor: cursor,
         **args
       ).each
+      enum = wrap(self, enum) if wrap
+      enum
     end
 
     def build_throttle_enumerator(enum, throttle_on:, backoff:)
@@ -136,6 +138,10 @@ module JobIteration
       ).to_enum
     end
 
+    def build_csv_enumerator(enumerable, cursor:)
+      CsvEnumerator.new(enumerable).rows(cursor: cursor)
+    end
+
     alias_method :once, :build_once_enumerator
     alias_method :times, :build_times_enumerator
     alias_method :array, :build_array_enumerator
@@ -143,6 +149,7 @@ module JobIteration
     alias_method :active_record_on_batches, :build_active_record_enumerator_on_batches
     alias_method :active_record_on_batch_relations, :build_active_record_enumerator_on_batch_relations
     alias_method :throttle, :build_throttle_enumerator
+    alias_method :csv, :build_csv_enumerator
 
     private
 

data/lib/job-iteration/integrations/resque.rb

@@ -10,8 +10,14 @@ module JobIteration
         super
       end
     end
-    # The patch is required in order to call shutdown? on a Resque::Worker instance
-    Resque::Worker.prepend(ResqueIterationExtension)
+
+    # @private
+    module ::Resque
+      class Worker
+        # The patch is required in order to call shutdown? on a Resque::Worker instance
+        prepend(ResqueIterationExtension)
+      end
+    end
 
     JobIteration.interruption_adapter = -> { $resque_worker.try!(:shutdown?) }
   end

data/lib/job-iteration/iteration.rb

@@ -68,6 +68,7 @@ module JobIteration
 
     def initialize(*arguments)
       super
+      @needs_reenqueue = false
       self.times_interrupted = 0
       self.total_time = 0.0
       assert_implements_methods!
@@ -132,7 +133,9 @@ module JobIteration
 
       run_callbacks(:shutdown)
 
-      if run_complete_callbacks?(completed)
+      if @needs_reenqueue
+        reenqueue_iteration_job
+      elsif run_complete_callbacks?(completed)
         run_callbacks(:complete)
         output_interrupt_summary
       end
@@ -141,6 +144,8 @@ module JobIteration
     def iterate_with_enumerator(enumerator, arguments)
       arguments = arguments.dup.freeze
       found_record = false
+      @needs_reenqueue = false
+
       enumerator.each do |object_from_enumerator, index|
         # Deferred until 2.0.0
         # assert_valid_cursor!(index)
@@ -153,7 +158,7 @@ module JobIteration
 
         next unless job_should_exit?
         self.executions -= 1 if executions > 1
-        reenqueue_iteration_job
+        @needs_reenqueue = true
         return false
       end
 
@@ -278,7 +283,7 @@ module JobIteration
     def valid_cursor_parameter?(parameters)
       # this condition is when people use the splat operator.
       # def build_enumerator(*)
-      return true if parameters == [[:rest]]
+      return true if parameters == [[:rest]] || parameters == [[:rest, :*]]
 
       parameters.each do |parameter_type, parameter_name|
         next unless parameter_name == :cursor

data/lib/job-iteration/throttle_enumerator.rb

@@ -28,14 +28,14 @@ module JobIteration
 
     def to_enum
       Enumerator.new(-> { @enum.size }) do |yielder|
-        loop do
+        @enum.each do |*val|
           if should_throttle?
             ActiveSupport::Notifications.instrument("throttled.iteration", job_class: @job.class.name)
             @job.retry_job(wait: @backoff)
             throw(:abort, :skip_complete_callbacks)
           end
 
-          yielder.yield(*@enum.next_values)
+          yielder.yield(*val)
         end
       end
     end

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.3.1"
+  VERSION = "1.3.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.3.5
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-11-11 00:00:00.000000000 Z
+date: 2022-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -85,7 +85,7 @@ homepage: https://github.com/shopify/job-iteration
 licenses:
 - MIT
 metadata:
-  changelog_uri: https://github.com/Shopify/job-iteration/blob/master/CHANGELOG.md
+  changelog_uri: https://github.com/Shopify/job-iteration/blob/main/CHANGELOG.md
   allowed_push_host: https://rubygems.org
 post_install_message: 
 rdoc_options: []