job-iteration 1.1.4 → 1.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 21dc74805b967b6082859b0c9a07d955f9f77ca3e82f250ceaf4f2aecf4afd7b
-  data.tar.gz: 23ea1a3e03c8b0578dcf852449204f60e5391bdb92f26987da7d42f45b052595
+  metadata.gz: 1a4821d55797f7597b3c916d6fbca2195c054bd3aab0ffcc40c9ce21ae0b1ec4
+  data.tar.gz: e71ce0fb7ec744b502f018ccfc7ea5e819bfe06f9fb2ca5a32c70cc1b875abc7
 SHA512:
-  metadata.gz: b3b5b27d999f8ea00a443d01857f0483fef67edcd0b0525d8ef8003c8814b9efe1e34713d6f45d0aa2de68c79c2517be141595f37c2501265f04ea6a4da7e0af
-  data.tar.gz: e47797081fc96034f2fe21746528047cba4f2873a0e6a36a7a38f2265d38a1dcb045f66216c3e9579699d63a9b6416136cf456e630404bb6d5381c433ef7232c
+  metadata.gz: 943795fabf60cff76fa7b8678f1f5d5a83bd98340194e142092dc891ff1abe991c0410572cebd09155c6c220ff96aa5530fb27e1b4fe119c6bbbf22fe12670ef
+  data.tar.gz: 11ea6f0165aa31350c6486236a33d66c94316a08f52905aeaab049388b433c229ccf52e69ae3e43eb8bca9a5c56dc4aae46803aae4add9d2a052d5d49af14d8d

data/.travis.yml

@@ -3,8 +3,9 @@ services:
   - redis-server
 language: ruby
 rvm:
-  - 2.5.5
-  - 2.6.2
+  - 2.5
+  - 2.6
+  - 2.7
 before_install:
   - mysql -e 'CREATE DATABASE job_iteration_test;'
 script:
@@ -14,4 +15,5 @@ script:
 
 gemfile:
   - 'gemfiles/rails_5_2.gemfile'
+  - 'gemfiles/rails_6_0.gemfile'
   - 'gemfiles/rails_edge.gemfile'

data/CHANGELOG.md

@@ -4,6 +4,27 @@
 
 #### Bug fix
 
+## v1.1.9 (January 6, 2021)
+
+- [61](https://github.com/Shopify/job-iteration/pull/61) Call `super` in `method_added`
+
+## v1.1.8 (June 8, 2020)
+
+- Preserve ruby2_keywords tags in arguments on Ruby 2.7
+
+## v1.1.7 (June 4, 2020)
+
+- [54](https://github.com/Shopify/job-iteration/pull/54) - Fix warnings on Ruby 2.7
+
+## v1.1.6 (May 22, 2020)
+
+- [49](https://github.com/Shopify/job-iteration/pull/49) -  Log when enumerator has nothing to iterate
+- [52](https://github.com/Shopify/job-iteration/pull/52) -  Fix CSVEnumerator cursor to properly remove already processed rows
+
+## v1.1.5 (February 27, 2020)
+
+- [47](https://github.com/Shopify/job-iteration/pull/47) -  Optional `sorbet-runtime` support for `JobIteration::Iteration` interface validation
+
 ## v1.1.4 (December 13, 2019)
 
 - [45](https://github.com/Shopify/job-iteration/pull/45) -  Add Throttle enumerator

data/Gemfile

@@ -23,3 +23,6 @@ gem 'mocha'
 gem 'rubocop', '~> 0.77.0'
 gem 'yard'
 gem 'rake'
+
+# for unit testing optional sorbet support
+gem 'sorbet-runtime'

data/README.md

@@ -9,7 +9,7 @@ Meet Iteration, an extension for [ActiveJob](https://github.com/rails/rails/tree
 Imagine the following job:
 
 ```ruby
-class SimpleJob < ActiveJob::Base
+class SimpleJob < ApplicationJob
   def perform
     User.find_each do |user|
       user.notify_about_something
@@ -43,7 +43,7 @@ And then execute:
 In the job, include `JobIteration::Iteration` module and start describing the job with two methods (`build_enumerator` and `each_iteration`) instead of `perform`:
 
 ```ruby
-class NotifyUsersJob < ActiveJob::Base
+class NotifyUsersJob < ApplicationJob
   include JobIteration::Iteration
 
   def build_enumerator(cursor:)
@@ -64,7 +64,9 @@ end
 Check out more examples of Iterations:
 
 ```ruby
-class BatchesJob < ActiveJob::Iteration
+class BatchesJob < ApplicationJob
+  include JobIteration::Iteration
+
   def build_enumerator(product_id, cursor:)
     enumerator_builder.active_record_on_batches(
       Product.find(product_id).comments,
@@ -81,7 +83,9 @@ end
 ```
 
 ```ruby
-class ArrayJob < ActiveJob::Iteration
+class ArrayJob < ApplicationJob
+  include JobIteration::Iteration
+
   def build_enumerator(cursor:)
     enumerator_builder.array(['build', 'enumerator', 'from', 'any', 'array'], cursor: cursor)
   end
@@ -93,7 +97,9 @@ end
 ```
 
 ```ruby
-class CsvJob < ActiveJob::Iteration
+class CsvJob < ApplicationJob
+  include JobIteration::Iteration
+
   def build_enumerator(import_id, cursor:)
     import = Import.find(import_id)
     JobIteration::CsvEnumerator.new(import.csv).rows(cursor: cursor)
@@ -153,7 +159,7 @@ There a few configuration assumptions that are required for Iteration to work wi
 **My job has a complex flow. How do I write my own Enumerator?** Iteration API takes care of persisting the cursor (that you may use to calculate an offset) and controlling the job state. The power of Enumerator object is that you can use the cursor in any way you want. One example is a cursorless job that pops records from a datastore until the job is interrupted:
 
 ```ruby
-class MyJob < ActiveJob::Base
+class MyJob < ApplicationJob
   include JobIteration::Iteration
 
   def build_enumerator(cursor:)

data/dev.yml

@@ -7,7 +7,7 @@ up:
     - mysql-client:
         or:        [mysql@5.7]
   - ruby:
-      version: 2.6.2
+      version: 2.6.5
   - railgun
   - bundler
   - custom:

data/gemfiles/rails_6_0.gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+eval_gemfile '../Gemfile'
+
+gem 'activejob', '~> 6.0.0'
+gem 'activerecord', '~> 6.0.0'

data/guides/custom-enumerator.md

@@ -8,7 +8,7 @@ class ListJob < ActiveJob::Base
 
   def build_enumerator(*)
     @redis = Redis.new
-    Enumerator.new |yielder|
+    Enumerator.new do |yielder|
       yielder.yield @redis.lpop(key), nil
     end
   end
@@ -72,3 +72,5 @@ end
 ```
 
 We recommend that you read the implementation of the other enumerators that come with the library (`CsvEnumerator`, `ActiveRecordEnumerator`) to gain a better understanding of building Enumerator objects.
+
+Code that is written after the `yield` in a custom enumerator is not guaranteed to execute. In the case that a job is forced to exit ie `job_should_exit?` is true, then the job is re-enqueued during the yield and the rest of the code in the enumerator does not run. You can follow that logic [here](https://github.com/Shopify/job-iteration/blob/9641f455b9126efff2214692c0bef423e0d12c39/lib/job-iteration/iteration.rb#L128-L131).

data/job-iteration.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = %w(lib)
 
   spec.metadata["changelog_uri"] = "https://github.com/Shopify/job-iteration/blob/master/CHANGELOG.md"
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
 
   spec.add_development_dependency("activerecord")
   spec.add_dependency("activejob", ">= 5.2")

data/lib/job-iteration/active_record_cursor.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 module JobIteration
+  # Curious about how this works from the SQL perspective?
+  # Check "Pagination Done the Right way": https://bit.ly/2Rq7iPF
   class ActiveRecordCursor # @private
     include Comparable
 

data/lib/job-iteration/csv_enumerator.rb

@@ -32,8 +32,8 @@ module JobIteration
     def rows(cursor:)
       @csv.lazy
         .each_with_index
-        .drop(cursor.to_i)
-        .to_enum { count_rows_in_file }
+        .drop(count_of_processed_rows(cursor))
+        .to_enum { count_of_rows_in_file }
     end
 
     # Constructs a enumerator on batches of CSV rows
@@ -42,13 +42,13 @@ module JobIteration
       @csv.lazy
         .each_slice(batch_size)
         .each_with_index
-        .drop(cursor.to_i)
-        .to_enum { (count_rows_in_file.to_f / batch_size).ceil }
+        .drop(count_of_processed_rows(cursor))
+        .to_enum { (count_of_rows_in_file.to_f / batch_size).ceil }
     end
 
     private
 
-    def count_rows_in_file
+    def count_of_rows_in_file
       # TODO: Remove rescue for NoMethodError when Ruby 2.6 is no longer supported.
       begin
         filepath = @csv.path
@@ -63,5 +63,9 @@ module JobIteration
       count -= 1 if @csv.headers
       count
     end
+
+    def count_of_processed_rows(cursor)
+      cursor.nil? ? 0 : cursor + 1
+    end
   end
 end

data/lib/job-iteration/iteration.rb

@@ -22,6 +22,7 @@ module JobIteration
     module ClassMethods
       def method_added(method_name)
         ban_perform_definition if method_name.to_sym == :perform
+        super
       end
 
       def on_start(*filters, &blk)
@@ -49,6 +50,7 @@ module JobIteration
       self.total_time = 0.0
       assert_implements_methods!
     end
+    ruby2_keywords(:initialize) if respond_to?(:ruby2_keywords, true)
 
     def serialize # @private
       super.merge(
@@ -69,7 +71,7 @@ module JobIteration
       interruptible_perform(*params)
     end
 
-    def retry_job(*)
+    def retry_job(*, **)
       super unless defined?(@retried) && @retried
       @retried = true
     end
@@ -116,8 +118,10 @@ module JobIteration
 
     def iterate_with_enumerator(enumerator, arguments)
       arguments = arguments.dup.freeze
+      found_record = false
       enumerator.each do |object_from_enumerator, index|
         record_unit_of_work do
+          found_record = true
           each_iteration(object_from_enumerator, *arguments)
           self.cursor_position = index
         end
@@ -128,6 +132,11 @@ module JobIteration
         return false
       end
 
+      logger.info(
+        "[JobIteration::Iteration] Enumerator found nothing to iterate! " \
+        "times_interrupted=#{times_interrupted} cursor_position=#{cursor_position}"
+      ) unless found_record
+
       true
     end
 
@@ -176,7 +185,7 @@ module JobIteration
       end
 
       if respond_to?(:build_enumerator, true)
-        parameters = method(:build_enumerator).parameters
+        parameters = method_parameters(:build_enumerator)
         unless valid_cursor_parameter?(parameters)
           raise ArgumentError, "Iteration job (#{self.class}) #build_enumerator " \
           "expects the keyword argument `cursor`"
@@ -187,6 +196,17 @@ module JobIteration
       end
     end
 
+    def method_parameters(method_name)
+      method = method(method_name)
+
+      if defined?(T::Private::Methods)
+        signature = T::Private::Methods.signature_for_method(method)
+        method = signature.method if signature
+      end
+
+      method.parameters
+    end
+
     def iteration_instrumentation_tags
       { job_class: self.class.name }
     end

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.1.4"
+  VERSION = "1.1.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.1.9
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-12-13 00:00:00.000000000 Z
+date: 2021-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -58,6 +58,7 @@ files:
 - bin/setup
 - dev.yml
 - gemfiles/rails_5_2.gemfile
+- gemfiles/rails_6_0.gemfile
 - gemfiles/rails_edge.gemfile
 - guides/best-practices.md
 - guides/custom-enumerator.md
@@ -81,6 +82,7 @@ licenses:
 - MIT
 metadata:
   changelog_uri: https://github.com/Shopify/job-iteration/blob/master/CHANGELOG.md
+  allowed_push_host: https://rubygems.org
 post_install_message: 
 rdoc_options: []
 require_paths: