job-iteration 1.2.0 → 1.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eff37be02274e3c0db10de1cf3e49a10eea5b5b838b19800d143853aee94c08f
-  data.tar.gz: '085dc8b7eec8897393461e43fb4ece8acaddc6705165a2b6de0856cef54b1215'
+  metadata.gz: d715ff6962fc9ff1081d5db93be694b0f0ed3e9fc2935b0d9f252971ea8a40d0
+  data.tar.gz: 60571b6b588816ad637a0eaaee238adb423c230102d4a73745e0fcc3b9e41b17
 SHA512:
-  metadata.gz: 22352326237ca16a23fc4177611187d4c6d426c07397caecd09165303726ce9ba00acaa882e35fb580b19bbe53f056ddde642298268a329dd02c88f61f82b5b7
-  data.tar.gz: 583406e5180f2721895673aa0410a390dce378c9c30e625151b5455fa8c1331867e94cc1cf2518b8b29cb0fbe908af517ee88a88429fd5f71f34a80617f5d8e6
+  metadata.gz: 587952d2f628c6d43c982ee8a4fe85d409d4d39044b34aa436bf97a2dd2679d92bfcb098bb2b35380b90297359dbefddc805a188501bd464665e8199a21f673e
+  data.tar.gz: dc172c2fa8172ee47d2af5ed7d47513b8e1131121f6b4f8deebf534a1c7f7e38e8b01f421bf3cc1e419aec9d1c5469020e690757be1e87c35e6d6dc0db8a437c

data/.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+
+updates:
+
+  - package-ecosystem: bundler
+    directory: '/'
+    versioning-strategy: increase
+    open-pull-requests-limit: 100
+    insecure-external-code-execution: allow
+    schedule:
+      interval: weekly
+
+  - package-ecosystem: github-actions
+    directory: '/'
+    schedule:
+      interval: daily

data/.gitignore

@@ -9,4 +9,3 @@
 .ruby-version
 .rubocop-http---shopify-github-io-ruby-style-guide-rubocop-yml
 gemfiles/*.lock
-Gemfile.lock

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 ### Master (unreleased)
 
+## v.1.3.3 (Nov 17, 2021)
+- [153](https://github.com/Shopify/job-iteration/pull/153) - Re-enqueue jobs only after shutdown hooks have run
+
+## v1.3.2 (Nov 12, 2021)
+- [148](https://github.com/Shopify/job-iteration/pull/148) - Revert "Do not evaluate enumerator when throttled", due to backwards incompatibility.
+
+## v1.3.1 (Nov 11, 2021)
+- [87](https://github.com/Shopify/job-iteration/pull/87) - Do not evaluate enumerator when throttled (REVERTED)
+
+
+## v1.3.0 (Oct 7, 2021)
+- [133](https://github.com/Shopify/job-iteration/pull/133) - Moves attributes out of JobIteration::Iteration included block
+
+
 ## v1.2.0 (Sept 21, 2021)
 - [107](https://github.com/Shopify/job-iteration/pull/107) - Remove broken links from README
 - [108](https://github.com/Shopify/job-iteration/pull/108) - Drop support for ruby 2.5

data/Gemfile.lock

@@ -0,0 +1,123 @@
+PATH
+  remote: .
+  specs:
+    job-iteration (1.3.3)
+      activejob (>= 5.2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activejob (6.1.4.1)
+      activesupport (= 6.1.4.1)
+      globalid (>= 0.3.6)
+    activemodel (6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activerecord (6.1.4.1)
+      activemodel (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activesupport (6.1.4.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    ast (2.4.2)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.9)
+    connection_pool (2.2.5)
+    database_cleaner (2.0.1)
+      database_cleaner-active_record (~> 2.0.0)
+    database_cleaner-active_record (2.0.1)
+      activerecord (>= 5.a)
+      database_cleaner-core (~> 2.0.0)
+    database_cleaner-core (2.0.1)
+    globalid (0.5.2)
+      activesupport (>= 5.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    method_source (1.0.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mono_logger (1.1.1)
+    multi_json (1.15.0)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
+    mysql2 (0.5.3)
+    parallel (1.21.0)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rack (2.2.3)
+    rack-protection (2.1.0)
+      rack
+    rainbow (3.0.0)
+    rake (13.0.6)
+    redis (4.5.1)
+    redis-namespace (1.8.1)
+      redis (>= 3.0.4)
+    regexp_parser (2.1.1)
+    resque (2.2.0)
+      mono_logger (~> 1.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.6)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    rexml (3.2.5)
+    rubocop (1.22.1)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.12.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.12.0)
+      parser (>= 3.0.1.1)
+    rubocop-shopify (2.3.0)
+      rubocop (~> 1.22)
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
+    sidekiq (6.3.1)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
+    sinatra (2.1.0)
+      mustermann (~> 1.0)
+      rack (~> 2.2)
+      rack-protection (= 2.1.0)
+      tilt (~> 2.0)
+    sorbet-runtime (0.5.9219)
+    tilt (2.0.10)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.1.0)
+    vegas (0.1.11)
+      rack (>= 1.0.0)
+    yard (0.9.26)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord
+  database_cleaner
+  globalid
+  i18n
+  job-iteration!
+  mocha
+  mysql2 (~> 0.5)
+  pry
+  rake
+  redis
+  resque
+  rubocop-shopify
+  sidekiq
+  sorbet-runtime
+  yard
+
+BUNDLED WITH
+   2.2.22

data/lib/job-iteration/active_record_batch_enumerator.rb

@@ -40,7 +40,7 @@ module JobIteration
     end
 
     def size
-      (@base_relation.count + @batch_size - 1) / @batch_size # ceiling division
+      (@base_relation.count(:all) + @batch_size - 1) / @batch_size # ceiling division
     end
 
     private

data/lib/job-iteration/active_record_cursor.rb

@@ -13,7 +13,7 @@ module JobIteration
       def initialize
         super(
           "The relation cannot use ORDER BY or LIMIT due to the way how iteration with a cursor is designed. " \
-          "You can use other ways to limit the number of rows, e.g. a WHERE condition on the primary key column."
+            "You can use other ways to limit the number of rows, e.g. a WHERE condition on the primary key column."
         )
       end
     end

data/lib/job-iteration/active_record_enumerator.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require_relative "./active_record_cursor"
 module JobIteration
   # Builds Enumerator based on ActiveRecord Relation. Supports enumerating on rows and batches.
@@ -33,7 +34,7 @@ module JobIteration
     end
 
     def size
-      @relation.count
+      @relation.count(:all)
     end
 
     private

data/lib/job-iteration/enumerator_builder.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require_relative "./active_record_batch_enumerator"
 require_relative "./active_record_enumerator"
 require_relative "./csv_enumerator"

data/lib/job-iteration/iteration.rb

@@ -6,6 +6,13 @@ module JobIteration
   module Iteration
     extend ActiveSupport::Concern
 
+    attr_accessor(
+      :cursor_position,
+      :start_time,
+      :times_interrupted,
+      :total_time,
+    )
+
     class CursorError < ArgumentError
       attr_reader :cursor
 
@@ -29,13 +36,6 @@ module JobIteration
     end
 
     included do |_base|
-      attr_accessor(
-        :cursor_position,
-        :start_time,
-        :times_interrupted,
-        :total_time,
-      )
-
       define_callbacks :start
       define_callbacks :shutdown
       define_callbacks :complete
@@ -68,6 +68,7 @@ module JobIteration
 
     def initialize(*arguments)
       super
+      @needs_reenqueue = false
       self.times_interrupted = 0
       self.total_time = 0.0
       assert_implements_methods!
@@ -132,7 +133,9 @@ module JobIteration
 
       run_callbacks(:shutdown)
 
-      if run_complete_callbacks?(completed)
+      if @needs_reenqueue
+        reenqueue_iteration_job
+      elsif run_complete_callbacks?(completed)
         run_callbacks(:complete)
         output_interrupt_summary
       end
@@ -141,6 +144,8 @@ module JobIteration
     def iterate_with_enumerator(enumerator, arguments)
       arguments = arguments.dup.freeze
       found_record = false
+      @needs_reenqueue = false
+
       enumerator.each do |object_from_enumerator, index|
         # Deferred until 2.0.0
         # assert_valid_cursor!(index)
@@ -153,13 +158,13 @@ module JobIteration
 
         next unless job_should_exit?
         self.executions -= 1 if executions > 1
-        reenqueue_iteration_job
+        @needs_reenqueue = true
         return false
       end
 
       logger.info(
         "[JobIteration::Iteration] Enumerator found nothing to iterate! " \
-        "times_interrupted=#{times_interrupted} cursor_position=#{cursor_position}"
+          "times_interrupted=#{times_interrupted} cursor_position=#{cursor_position}"
       ) unless found_record
 
       adjust_total_time
@@ -167,10 +172,8 @@ module JobIteration
       true
     end
 
-    def record_unit_of_work
-      ActiveSupport::Notifications.instrument("each_iteration.iteration", iteration_instrumentation_tags) do
-        yield
-      end
+    def record_unit_of_work(&block)
+      ActiveSupport::Notifications.instrument("each_iteration.iteration", iteration_instrumentation_tags, &block)
     end
 
     def reenqueue_iteration_job
@@ -210,7 +213,7 @@ module JobIteration
 
       raise CursorError.new(
         "Cursor must be composed of objects capable of built-in (de)serialization: " \
-        "Strings, Integers, Floats, Arrays, Hashes, true, false, or nil.",
+          "Strings, Integers, Floats, Arrays, Hashes, true, false, or nil.",
         cursor: cursor,
       )
     end
@@ -227,7 +230,7 @@ module JobIteration
         parameters = method_parameters(:build_enumerator)
         unless valid_cursor_parameter?(parameters)
           raise ArgumentError, "Iteration job (#{self.class}) #build_enumerator " \
-          "expects the keyword argument `cursor`"
+            "expects the keyword argument `cursor`"
         end
       else
         raise ArgumentError, "Iteration job (#{self.class}) must implement #build_enumerator " \

data/lib/job-iteration/throttle_enumerator.rb

@@ -1,5 +1,6 @@
 # typed: true
 # frozen_string_literal: true
+
 module JobIteration
   # ThrottleEnumerator allows you to throttle iterations
   # based on external signal (e.g. database health).

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.2.0"
+  VERSION = "1.3.3"
 end

data/lib/job-iteration.rb

@@ -22,6 +22,7 @@ module JobIteration
 
   # Used internally for hooking into job processing frameworks like Sidekiq and Resque.
   attr_accessor :interruption_adapter
+
   self.interruption_adapter = -> { false }
 
   # Set if you want to use your own enumerator builder instead of default EnumeratorBuilder.
@@ -33,6 +34,7 @@ module JobIteration
   #
   #   JobIteration.enumerator_builder = MyOwnBuilder
   attr_accessor :enumerator_builder
+
   self.enumerator_builder = JobIteration::EnumeratorBuilder
 
   def load_integrations
@@ -42,7 +44,7 @@ module JobIteration
       if loaded
         raise IntegrationLoadError,
           "#{loaded} integration has already been loaded, but #{integration} is also available. " \
-          "Iteration will only work with one integration."
+            "Iteration will only work with one integration."
       end
       loaded = integration
     rescue LoadError

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.3
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-09-22 00:00:00.000000000 Z
+date: 2021-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -45,6 +45,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
 - ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rubocop.yml"
@@ -52,6 +53,7 @@ files:
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile