job-iteration 1.3.0 → 1.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9b36e89b6fb57143e002ec9185093367f97b1e8db3b4e585ef2a5ca25a147008
-  data.tar.gz: 616a27066510613ce74de5cd76d9920188006377fbcfb2b56b1fdd8d045ba1a6
+  metadata.gz: 5d279ff6eabe074dc75f9907a5e733d5bac2da3b97a6657fe2b1bc9eb6f03554
+  data.tar.gz: 59c7b39c5351e21ae50cee2234d169a8c0e86014f4294c3b942f76402056eabf
 SHA512:
-  metadata.gz: 643cd18e4ed677c44bb5e709f286ffd20bd93602064967fa28cbf5db865290da7b4ada2619103ae3f34f9b8e6048684c652465e72d34513526515953dc1b1401
-  data.tar.gz: cdec5872bfd3096782ff954434c0ffaa723c9e7abf3d3136d8eb9e5486be20b067133cc9b5fcc162a5e460f38e80290036b67c4a6710779ed34410e99fb33996
+  metadata.gz: '021873185a818abe4f4c42744404239e8b0516c27f12fe028b0ed82920d5a1f595caf72a57955354c847575dc18671b8b5c47a2acd2c4b9cd24de153f916ebb2'
+  data.tar.gz: 1cbf2f0da940592a227ca922af2bc956be2c7f9ae68fb995f0f56d2a14f071983b37dc3726d364b10108e4344ae9f3e89ccaf88dae08b16b912ebf07ee110ddf

data/.github/workflows/ci.yml

@@ -13,13 +13,18 @@ jobs:
         - 6379:6379
     strategy:
       matrix:
-        ruby: [2.6, 2.7, 3.0]
+        ruby: ["2.6", "2.7", "3.0"]
         gemfile: [rails_5_2, rails_6_0, rails_edge]
         exclude:
-          - ruby: 2.6
+          - ruby: "2.6"
             gemfile: rails_edge
-          - ruby: 3.0
+          - ruby: "3.0"
             gemfile: rails_5_2
+        include:
+          - ruby: "3.1"
+            gemfile: rails_edge
+          - ruby: head
+            gemfile: rails_edge
     env:
       BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
     steps:

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 ### Master (unreleased)
 
+## v.1.3.4 (Jan 18, 2022)
+
+- [174](https://github.com/Shopify/job-iteration/pull/174) - Fix Ruby 3.2 compatibility
+
+## v.1.3.3 (Nov 17, 2021)
+- [153](https://github.com/Shopify/job-iteration/pull/153) - Re-enqueue jobs only after shutdown hooks have run
+
+## v1.3.2 (Nov 12, 2021)
+- [148](https://github.com/Shopify/job-iteration/pull/148) - Revert "Do not evaluate enumerator when throttled", due to backwards incompatibility.
+
+## v1.3.1 (Nov 11, 2021)
+- [87](https://github.com/Shopify/job-iteration/pull/87) - Do not evaluate enumerator when throttled (REVERTED)
+
+
 ## v1.3.0 (Oct 7, 2021)
 - [133](https://github.com/Shopify/job-iteration/pull/133) - Moves attributes out of JobIteration::Iteration included block
 

data/Gemfile

@@ -11,7 +11,7 @@ gemspec
 gem "sidekiq"
 gem "resque"
 
-gem "mysql2", "~> 0.5"
+gem "mysql2", github: "brianmario/mysql2"
 gem "globalid"
 gem "i18n"
 gem "redis"

data/Gemfile.lock

@@ -1,21 +1,27 @@
+GIT
+  remote: https://github.com/brianmario/mysql2
+  revision: 25c42c712118b046eb9df7a0f50ffde1a04ee6d1
+  specs:
+    mysql2 (0.5.3)
+
 PATH
   remote: .
   specs:
-    job-iteration (1.3.0)
+    job-iteration (1.3.4)
       activejob (>= 5.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (6.1.4.1)
-      activesupport (= 6.1.4.1)
+    activejob (6.1.4.4)
+      activesupport (= 6.1.4.4)
       globalid (>= 0.3.6)
-    activemodel (6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activerecord (6.1.4.1)
-      activemodel (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activesupport (6.1.4.1)
+    activemodel (6.1.4.4)
+      activesupport (= 6.1.4.4)
+    activerecord (6.1.4.4)
+      activemodel (= 6.1.4.4)
+      activesupport (= 6.1.4.4)
+    activesupport (6.1.4.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -31,20 +37,19 @@ GEM
       activerecord (>= 5.a)
       database_cleaner-core (~> 2.0.0)
     database_cleaner-core (2.0.1)
-    globalid (0.5.2)
+    globalid (1.0.0)
       activesupport (>= 5.0)
-    i18n (1.8.10)
+    i18n (1.8.11)
       concurrent-ruby (~> 1.0)
     method_source (1.0.0)
-    minitest (5.14.4)
+    minitest (5.15.0)
     mocha (1.13.0)
     mono_logger (1.1.1)
     multi_json (1.15.0)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
-    mysql2 (0.5.3)
     parallel (1.21.0)
-    parser (3.0.2.0)
+    parser (3.1.0.0)
       ast (~> 2.4.1)
     pry (0.14.1)
       coderay (~> 1.1)
@@ -52,35 +57,35 @@ GEM
     rack (2.2.3)
     rack-protection (2.1.0)
       rack
-    rainbow (3.0.0)
+    rainbow (3.1.1)
     rake (13.0.6)
-    redis (4.4.0)
+    redis (4.5.1)
     redis-namespace (1.8.1)
       redis (>= 3.0.4)
-    regexp_parser (2.1.1)
-    resque (2.1.0)
+    regexp_parser (2.2.0)
+    resque (2.2.0)
       mono_logger (~> 1.0)
       multi_json (~> 1.0)
       redis-namespace (~> 1.6)
       sinatra (>= 0.9.2)
       vegas (~> 0.1.2)
     rexml (3.2.5)
-    rubocop (1.22.1)
+    rubocop (1.24.1)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.12.0, < 2.0)
+      rubocop-ast (>= 1.15.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.12.0)
+    rubocop-ast (1.15.1)
       parser (>= 3.0.1.1)
-    rubocop-shopify (2.3.0)
-      rubocop (~> 1.22)
+    rubocop-shopify (2.4.0)
+      rubocop (~> 1.24)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.5)
-    sidekiq (6.2.2)
+    sidekiq (6.3.1)
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
       redis (>= 4.2.0)
@@ -89,7 +94,7 @@ GEM
       rack (~> 2.2)
       rack-protection (= 2.1.0)
       tilt (~> 2.0)
-    sorbet-runtime (0.5.9158)
+    sorbet-runtime (0.5.9426)
     tilt (2.0.10)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
@@ -97,7 +102,7 @@ GEM
     vegas (0.1.11)
       rack (>= 1.0.0)
     yard (0.9.26)
-    zeitwerk (2.4.2)
+    zeitwerk (2.5.1)
 
 PLATFORMS
   ruby
@@ -109,7 +114,7 @@ DEPENDENCIES
   i18n
   job-iteration!
   mocha
-  mysql2 (~> 0.5)
+  mysql2!
   pry
   rake
   redis
@@ -120,4 +125,4 @@ DEPENDENCIES
   yard
 
 BUNDLED WITH
-   2.2.20
+   2.3.5

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018 Shopify
+Copyright (c) 2018-present Shopify
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -2,7 +2,7 @@
 
 [![CI](https://github.com/Shopify/job-iteration/actions/workflows/ci.yml/badge.svg)](https://github.com/Shopify/job-iteration/actions/workflows/ci.yml)
 
-Meet Iteration, an extension for [ActiveJob](https://github.com/rails/rails/tree/master/activejob) that makes your jobs interruptible and resumable, saving all progress that the job has made (aka checkpoint for jobs).
+Meet Iteration, an extension for [ActiveJob](https://github.com/rails/rails/tree/main/activejob) that makes your jobs interruptible and resumable, saving all progress that the job has made (aka checkpoint for jobs).
 
 ## Background
 
@@ -223,4 +223,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Job::Iteration project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/Shopify/job-iteration/blob/master/CODE_OF_CONDUCT.md).
+Everyone interacting in the Job::Iteration project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/Shopify/job-iteration/blob/main/CODE_OF_CONDUCT.md).

data/guides/best-practices.md

@@ -41,6 +41,14 @@ ActiveSupport::Notifications.subscribe('interrupted.iteration') do |_, _, _, _,
     tags: { job_class: tags[:job_class]&.underscore }
   )
 end
+
+# If you're using ThrottleEnumerator
+ActiveSupport::Notifications.subscribe('throttled.iteration') do |_, _, _, _, tags|
+  StatsD.increment(
+    "iteration.throttled",
+    tags: { job_class: tags[:job_class]&.underscore }
+  )
+end
 ```
 
 ## Max iteration time

data/job-iteration.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.metadata["changelog_uri"] = "https://github.com/Shopify/job-iteration/blob/master/CHANGELOG.md"
+  spec.metadata["changelog_uri"] = "https://github.com/Shopify/job-iteration/blob/main/CHANGELOG.md"
   spec.metadata["allowed_push_host"] = "https://rubygems.org"
 
   spec.add_development_dependency("activerecord")

data/lib/job-iteration/active_record_batch_enumerator.rb

@@ -40,7 +40,7 @@ module JobIteration
     end
 
     def size
-      (@base_relation.count + @batch_size - 1) / @batch_size # ceiling division
+      (@base_relation.count(:all) + @batch_size - 1) / @batch_size # ceiling division
     end
 
     private
@@ -65,7 +65,8 @@ module JobIteration
       @cursor = Array.wrap(cursor)
 
       # Yields relations by selecting the primary keys of records in the batch.
-      # Post.where(published: nil) results in an enumerator of relations like: Post.where(ids: batch_of_ids)
+      # Post.where(published: nil) results in an enumerator of relations like:
+      # Post.where(published: nil, ids: batch_of_ids)
       @base_relation.where(@primary_key => ids)
     end
 

data/lib/job-iteration/active_record_enumerator.rb

@@ -34,7 +34,7 @@ module JobIteration
     end
 
     def size
-      @relation.count
+      @relation.count(:all)
     end
 
     private

data/lib/job-iteration/enumerator_builder.rb

@@ -119,12 +119,14 @@ module JobIteration
 
     # Builds Enumerator from Active Record Relation and enumerates on batches, yielding Active Record Relations.
     # See documentation for #build_active_record_enumerator_on_batches.
-    def build_active_record_enumerator_on_batch_relations(scope, cursor:, **args)
-      JobIteration::ActiveRecordBatchEnumerator.new(
+    def build_active_record_enumerator_on_batch_relations(scope, wrap: true, cursor:, **args)
+      enum = JobIteration::ActiveRecordBatchEnumerator.new(
         scope,
         cursor: cursor,
         **args
       ).each
+      enum = wrap(self, enum) if wrap
+      enum
     end
 
     def build_throttle_enumerator(enum, throttle_on:, backoff:)

data/lib/job-iteration/integrations/resque.rb

@@ -10,8 +10,14 @@ module JobIteration
         super
       end
     end
-    # The patch is required in order to call shutdown? on a Resque::Worker instance
-    Resque::Worker.prepend(ResqueIterationExtension)
+
+    # @private
+    module ::Resque
+      class Worker
+        # The patch is required in order to call shutdown? on a Resque::Worker instance
+        prepend(ResqueIterationExtension)
+      end
+    end
 
     JobIteration.interruption_adapter = -> { $resque_worker.try!(:shutdown?) }
   end

data/lib/job-iteration/iteration.rb

@@ -68,6 +68,7 @@ module JobIteration
 
     def initialize(*arguments)
       super
+      @needs_reenqueue = false
       self.times_interrupted = 0
       self.total_time = 0.0
       assert_implements_methods!
@@ -132,7 +133,9 @@ module JobIteration
 
       run_callbacks(:shutdown)
 
-      if run_complete_callbacks?(completed)
+      if @needs_reenqueue
+        reenqueue_iteration_job
+      elsif run_complete_callbacks?(completed)
         run_callbacks(:complete)
         output_interrupt_summary
       end
@@ -141,6 +144,8 @@ module JobIteration
     def iterate_with_enumerator(enumerator, arguments)
       arguments = arguments.dup.freeze
       found_record = false
+      @needs_reenqueue = false
+
       enumerator.each do |object_from_enumerator, index|
         # Deferred until 2.0.0
         # assert_valid_cursor!(index)
@@ -153,7 +158,7 @@ module JobIteration
 
         next unless job_should_exit?
         self.executions -= 1 if executions > 1
-        reenqueue_iteration_job
+        @needs_reenqueue = true
         return false
       end
 
@@ -278,7 +283,7 @@ module JobIteration
     def valid_cursor_parameter?(parameters)
       # this condition is when people use the splat operator.
       # def build_enumerator(*)
-      return true if parameters == [[:rest]]
+      return true if parameters == [[:rest]] || parameters == [[:rest, :*]]
 
       parameters.each do |parameter_type, parameter_name|
         next unless parameter_name == :cursor

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.3.0"
+  VERSION = "1.3.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.3.4
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-10-08 00:00:00.000000000 Z
+date: 2022-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -85,7 +85,7 @@ homepage: https://github.com/shopify/job-iteration
 licenses:
 - MIT
 metadata:
-  changelog_uri: https://github.com/Shopify/job-iteration/blob/master/CHANGELOG.md
+  changelog_uri: https://github.com/Shopify/job-iteration/blob/main/CHANGELOG.md
   allowed_push_host: https://rubygems.org
 post_install_message: 
 rdoc_options: []