job-iteration 1.1.13 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c1707731e6bf03617bdbaa60bc2ddeb9652f34534a8fe0d69b26e233c790d469
-  data.tar.gz: a981f1230a7784e4e6673c0d2e4c0642f4a1d3c48baeb3039ca6e41daced9107
+  metadata.gz: 8788b6f2da0dba53c31c57d529a371e3e822091caa3654bcafc6cf25ad031adb
+  data.tar.gz: 14cba8e6969be4d6aba0f729abd355196324f2eebd00d9b659053b7efe25117d
 SHA512:
-  metadata.gz: db396697d25d4b0450b69d2a58d68e1e247a074f02c9a5887746184789f2aca2569e345ef0d88fb5e142fcaa1c0586169ef91b7bc0ffe2ef35796f544e0cafa1
-  data.tar.gz: 757b0c074830cbe97d49fd6e84c59ade45c4b5ecdc57b52cf17ea5539e9f2bbffb8ea929804da531c217bf43bbc4f67f726e9d0f1a9a391e32e35837b7edf624
+  metadata.gz: 97f3d383f9eb1e82c3c68ee21bdcaf31a5df2923ebd4eb9089b7b78348f64bd69aa0da39acd0eba23cd309263931639e4e6ca702511a0d83a9528e520ff77e89
+  data.tar.gz: beda68f91cfd51d0d78c2b0c26b20b70bc339fc25591d3ceefe825e687bdc057b448832d2abead1e80c5259eda52ef04d6b6cf83063431766ab8920dd753c396

data/.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+
+updates:
+
+  - package-ecosystem: bundler
+    directory: '/'
+    versioning-strategy: increase
+    open-pull-requests-limit: 100
+    insecure-external-code-execution: allow
+    schedule:
+      interval: weekly
+
+  - package-ecosystem: github-actions
+    directory: '/'
+    schedule:
+      interval: daily

data/.github/workflows/ci.yml

@@ -13,11 +13,9 @@ jobs:
         - 6379:6379
     strategy:
       matrix:
-        ruby: [2.5, 2.6, 2.7, 3.0]
+        ruby: [2.6, 2.7, 3.0]
         gemfile: [rails_5_2, rails_6_0, rails_edge]
         exclude:
-          - ruby: 2.5
-            gemfile: rails_edge
           - ruby: 2.6
             gemfile: rails_edge
           - ruby: 3.0

data/.gitignore

@@ -9,4 +9,3 @@
 .ruby-version
 .rubocop-http---shopify-github-io-ruby-style-guide-rubocop-yml
 gemfiles/*.lock
-Gemfile.lock

data/.rubocop.yml

@@ -2,7 +2,7 @@ inherit_gem:
   rubocop-shopify: rubocop.yml
 
 AllCops:
-  TargetRubyVersion: 2.4.4
+  TargetRubyVersion: 2.6.5
   Exclude:
     - 'vendor/bundle/**/*'
 Lint/SuppressedException:

data/CHANGELOG.md

@@ -1,5 +1,26 @@
 ### Master (unreleased)
 
+## v1.3.1 (Nov 11, 2021)
+- [87](https://github.com/Shopify/job-iteration/pull/87) - Do not evaluate enumerator when throttled
+
+
+## v1.3.0 (Oct 7, 2021)
+- [133](https://github.com/Shopify/job-iteration/pull/133) - Moves attributes out of JobIteration::Iteration included block
+
+
+## v1.2.0 (Sept 21, 2021)
+- [107](https://github.com/Shopify/job-iteration/pull/107) - Remove broken links from README
+- [108](https://github.com/Shopify/job-iteration/pull/108) - Drop support for ruby 2.5
+- [110](https://github.com/Shopify/job-iteration/pull/110) - Update rubocop TargetRubyVersion
+
+## v1.1.14 (May 28, 2021)
+
+#### Bug fix
+- [84](https://github.com/Shopify/job-iteration/pull/84) - Call adjust_total_time before running on_complete callbacks
+- [94](https://github.com/Shopify/job-iteration/pull/94) - Remove unnecessary break
+- [95](https://github.com/Shopify/job-iteration/pull/95) - ActiveRecordBatchEnumerator#each should rewind at the end
+- [97](https://github.com/Shopify/job-iteration/pull/97) - Batch enumerator size returns the number of batches, not records
+
 ## v1.1.13 (May 20, 2021)
 
 #### New feature

data/Gemfile

@@ -21,7 +21,6 @@ gem "pry"
 gem "mocha"
 
 gem "rubocop-shopify", require: false
-gem "rubocop", "<= 1.12.1", require: false # 1.13.0 drops Ruby 2.4 support
 gem "yard"
 gem "rake"
 

data/Gemfile.lock

@@ -0,0 +1,123 @@
+PATH
+  remote: .
+  specs:
+    job-iteration (1.3.1)
+      activejob (>= 5.2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activejob (6.1.4.1)
+      activesupport (= 6.1.4.1)
+      globalid (>= 0.3.6)
+    activemodel (6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activerecord (6.1.4.1)
+      activemodel (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activesupport (6.1.4.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    ast (2.4.2)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.9)
+    connection_pool (2.2.5)
+    database_cleaner (2.0.1)
+      database_cleaner-active_record (~> 2.0.0)
+    database_cleaner-active_record (2.0.1)
+      activerecord (>= 5.a)
+      database_cleaner-core (~> 2.0.0)
+    database_cleaner-core (2.0.1)
+    globalid (0.5.2)
+      activesupport (>= 5.0)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    method_source (1.0.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mono_logger (1.1.1)
+    multi_json (1.15.0)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
+    mysql2 (0.5.3)
+    parallel (1.21.0)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rack (2.2.3)
+    rack-protection (2.1.0)
+      rack
+    rainbow (3.0.0)
+    rake (13.0.6)
+    redis (4.5.1)
+    redis-namespace (1.8.1)
+      redis (>= 3.0.4)
+    regexp_parser (2.1.1)
+    resque (2.1.0)
+      mono_logger (~> 1.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.6)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    rexml (3.2.5)
+    rubocop (1.22.1)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.12.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.12.0)
+      parser (>= 3.0.1.1)
+    rubocop-shopify (2.3.0)
+      rubocop (~> 1.22)
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
+    sidekiq (6.2.2)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
+    sinatra (2.1.0)
+      mustermann (~> 1.0)
+      rack (~> 2.2)
+      rack-protection (= 2.1.0)
+      tilt (~> 2.0)
+    sorbet-runtime (0.5.9219)
+    tilt (2.0.10)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.1.0)
+    vegas (0.1.11)
+      rack (>= 1.0.0)
+    yard (0.9.26)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord
+  database_cleaner
+  globalid
+  i18n
+  job-iteration!
+  mocha
+  mysql2 (~> 0.5)
+  pry
+  rake
+  redis
+  resque
+  rubocop-shopify
+  sidekiq
+  sorbet-runtime
+  yard
+
+BUNDLED WITH
+   2.2.20

data/README.md

@@ -1,6 +1,6 @@
 # Job Iteration API
 
-[![Build Status](https://travis-ci.com/Shopify/job-iteration.svg?branch=master)](https://travis-ci.com/Shopify/job-iteration)
+[![CI](https://github.com/Shopify/job-iteration/actions/workflows/ci.yml/badge.svg)](https://github.com/Shopify/job-iteration/actions/workflows/ci.yml)
 
 Meet Iteration, an extension for [ActiveJob](https://github.com/rails/rails/tree/master/activejob) that makes your jobs interruptible and resumable, saving all progress that the job has made (aka checkpoint for jobs).
 
@@ -171,7 +171,7 @@ There a few configuration assumptions that are required for Iteration to work wi
 
 **Why is it important that `each_iteration` takes less than 30 seconds?** When the job worker is scheduled for restart or shutdown, it gets a notice to finish remaining unit of work. To guarantee that no progress is lost we need to make sure that `each_iteration` completes within a reasonable amount of time.
 
-**What do I do if each iteration takes a long time, because it's doing nested operations?** If your `each_iteration` is complex, we recommend enqueuing another job, which will run your nested business logic. We may expose primitives in the future to do this more effectively, but this is not terribly common today. We recommend to read https://goo.gl/UobaaU to learn more about nested operations.
+**What do I do if each iteration takes a long time, because it's doing nested operations?** If your `each_iteration` is complex, we recommend enqueuing another job, which will run your nested business logic. We may expose primitives in the future to do this more effectively, but this is not terribly common today.
 
 **Why do I use have to use this ugly helper in `build_enumerator`? Why can't you automatically infer it?** This is how the first version of the API worked. We checked the type of object returned by `build_enumerable`, and whether it was ActiveRecord Relation or an Array, we used the matching adapter. This caused opaque type branching in Iteration internals and it didn’t allow developers to craft their own Enumerators and control the cursor value. We made a decision to _always_ return Enumerator instance from `build_enumerator`. Now we provide explicit helpers to convert ActiveRecord Relation or an Array to Enumerator, and for more complex iteration flows developers can build their own `Enumerator` objects.
 

data/bin/test

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+def main
+  begin
+    command = create_command
+  rescue ArgumentError => e
+    abort(e.message)
+  end
+  puts "Running #{command.join(" ")}"
+  system(*command)
+end
+
+def create_command
+  case ARGV.length
+  when 0
+    ["bundle", "exec", "rake", "test"]
+  when 1
+    filename = ARGV[0]
+    ["bundle", "exec", "rake", "test", "TEST=#{filename}"]
+  when 2
+    filename = ARGV[0]
+    test_name = ARGV[1]
+    test_name_with_underscores = test_name.tr(" ", "_")
+    test_name_pattern = "/#{Regexp.escape(test_name_with_underscores)}/"
+    ["bundle", "exec", "rake", "test", "TEST=#{filename}", "TESTOPTS=\"--name=#{test_name_pattern} -v\""]
+  else
+    raise ArgumentError, "Too many arguments. Did you forget to put the test name in quotes?"
+  end
+end
+
+main

data/dev.yml

@@ -13,7 +13,43 @@ up:
   - custom:
       name: Create Job Iteration database
       meet: mysql -uroot -h job-iteration.railgun -e "CREATE DATABASE job_iteration_test"
-      met?: mysql -uroot -h job-iteration.railgun job_iteration_test -e "SELECT 1"
+      met?: mysql -uroot -h job-iteration.railgun job_iteration_test -e "SELECT 1" &> /dev/null
 
 commands:
-  test: bundle exec rake
+  test:
+    run:  bin/test "$@"
+    syntax:
+      optional: filename testnamepattern
+    aliases: [t]
+    desc: run tests
+    long_desc: |
+      {{bold:Default}}
+      =======
+      Run the entire test suite.
+
+        Examples:
+        {{command:dev test}}
+        {{command:dev t}}
+
+      {{bold:Run all tests in a file}}
+      ========================
+      Include the file path.
+
+        Example:
+        {{command:dev test test/unit/iteration_test.rb}}
+
+      {{bold:Run a single test in a given file}}
+      ========================
+      Include the file path and the name of the test you'd like to run.
+
+        Example:
+        {{command:dev test test/unit/iteration_test.rb test_that_it_has_a_version_number}}
+
+      {{bold:Run all tests in a given file whose name contains a string}}
+      ========================
+      Include the file path and the string that the test names should contain.
+
+        Example:
+        {{command:dev test test/unit/iteration_test.rb version_number}}
+  style:
+    run: bundle exec rubocop -a

data/job-iteration.gemspec

@@ -5,9 +5,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "job-iteration/version"
 
 Gem::Specification.new do |spec|
+  spec.required_ruby_version = ">= 2.6"
   spec.name          = "job-iteration"
   spec.version       = JobIteration::VERSION
-  spec.authors       = %w(Shopify)
+  spec.authors       = ["Shopify"]
   spec.email         = ["ops-accounts+shipit@shopify.com"]
 
   spec.summary       = "Makes your background jobs interruptible and resumable."
@@ -20,7 +21,7 @@ Gem::Specification.new do |spec|
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = %w(lib)
+  spec.require_paths = ["lib"]
 
   spec.metadata["changelog_uri"] = "https://github.com/Shopify/job-iteration/blob/master/CHANGELOG.md"
   spec.metadata["allowed_push_host"] = "https://rubygems.org"

data/lib/job-iteration/active_record_batch_enumerator.rb

@@ -19,6 +19,7 @@ module JobIteration
         @columns.dup << @primary_key
       end
       @cursor = Array.wrap(cursor)
+      @initial_cursor = @cursor
       raise ArgumentError, "Must specify at least one column" if @columns.empty?
       if relation.joins_values.present? && !@columns.all? { |column| column.to_s.include?(".") }
         raise ArgumentError, "You need to specify fully-qualified columns if you join a table"
@@ -34,13 +35,12 @@ module JobIteration
     def each
       return to_enum { size } unless block_given?
       while (relation = next_batch)
-        break if @cursor.nil?
         yield relation, cursor_value
       end
     end
 
     def size
-      @base_relation.count
+      (@base_relation.count + @batch_size - 1) / @batch_size # ceiling division
     end
 
     private
@@ -56,7 +56,10 @@ module JobIteration
       end
 
       cursor = cursor_values.last
-      return unless cursor.present?
+      unless cursor.present?
+        @cursor = @initial_cursor
+        return
+      end
       # The primary key was plucked, but original cursor did not include it, so we should remove it
       cursor.pop unless @primary_key_index
       @cursor = Array.wrap(cursor)

data/lib/job-iteration/active_record_cursor.rb

@@ -13,7 +13,7 @@ module JobIteration
       def initialize
         super(
           "The relation cannot use ORDER BY or LIMIT due to the way how iteration with a cursor is designed. " \
-          "You can use other ways to limit the number of rows, e.g. a WHERE condition on the primary key column."
+            "You can use other ways to limit the number of rows, e.g. a WHERE condition on the primary key column."
         )
       end
     end

data/lib/job-iteration/active_record_enumerator.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require_relative "./active_record_cursor"
 module JobIteration
   # Builds Enumerator based on ActiveRecord Relation. Supports enumerating on rows and batches.

data/lib/job-iteration/enumerator_builder.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require_relative "./active_record_batch_enumerator"
 require_relative "./active_record_enumerator"
 require_relative "./csv_enumerator"

data/lib/job-iteration/iteration.rb

@@ -6,6 +6,13 @@ module JobIteration
   module Iteration
     extend ActiveSupport::Concern
 
+    attr_accessor(
+      :cursor_position,
+      :start_time,
+      :times_interrupted,
+      :total_time,
+    )
+
     class CursorError < ArgumentError
       attr_reader :cursor
 
@@ -29,13 +36,6 @@ module JobIteration
     end
 
     included do |_base|
-      attr_accessor(
-        :cursor_position,
-        :start_time,
-        :times_interrupted,
-        :total_time,
-      )
-
       define_callbacks :start
       define_callbacks :shutdown
       define_callbacks :complete
@@ -159,16 +159,16 @@ module JobIteration
 
       logger.info(
         "[JobIteration::Iteration] Enumerator found nothing to iterate! " \
-        "times_interrupted=#{times_interrupted} cursor_position=#{cursor_position}"
+          "times_interrupted=#{times_interrupted} cursor_position=#{cursor_position}"
       ) unless found_record
 
+      adjust_total_time
+
       true
     end
 
-    def record_unit_of_work
-      ActiveSupport::Notifications.instrument("each_iteration.iteration", iteration_instrumentation_tags) do
-        yield
-      end
+    def record_unit_of_work(&block)
+      ActiveSupport::Notifications.instrument("each_iteration.iteration", iteration_instrumentation_tags, &block)
     end
 
     def reenqueue_iteration_job
@@ -208,7 +208,7 @@ module JobIteration
 
       raise CursorError.new(
         "Cursor must be composed of objects capable of built-in (de)serialization: " \
-        "Strings, Integers, Floats, Arrays, Hashes, true, false, or nil.",
+          "Strings, Integers, Floats, Arrays, Hashes, true, false, or nil.",
         cursor: cursor,
       )
     end
@@ -225,7 +225,7 @@ module JobIteration
         parameters = method_parameters(:build_enumerator)
         unless valid_cursor_parameter?(parameters)
           raise ArgumentError, "Iteration job (#{self.class}) #build_enumerator " \
-          "expects the keyword argument `cursor`"
+            "expects the keyword argument `cursor`"
         end
       else
         raise ArgumentError, "Iteration job (#{self.class}) must implement #build_enumerator " \
@@ -249,8 +249,6 @@ module JobIteration
     end
 
     def output_interrupt_summary
-      adjust_total_time
-
       message = "[JobIteration::Iteration] Completed iterating. times_interrupted=%d total_time=%.3f"
       logger.info(Kernel.format(message, times_interrupted, total_time))
     end

data/lib/job-iteration/throttle_enumerator.rb

@@ -1,5 +1,6 @@
 # typed: true
 # frozen_string_literal: true
+
 module JobIteration
   # ThrottleEnumerator allows you to throttle iterations
   # based on external signal (e.g. database health).
@@ -27,14 +28,14 @@ module JobIteration
 
     def to_enum
       Enumerator.new(-> { @enum.size }) do |yielder|
-        @enum.each do |*val|
+        loop do
           if should_throttle?
             ActiveSupport::Notifications.instrument("throttled.iteration", job_class: @job.class.name)
             @job.retry_job(wait: @backoff)
             throw(:abort, :skip_complete_callbacks)
           end
 
-          yielder.yield(*val)
+          yielder.yield(*@enum.next_values)
         end
       end
     end

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.1.13"
+  VERSION = "1.3.1"
 end

data/lib/job-iteration.rb

@@ -22,6 +22,7 @@ module JobIteration
 
   # Used internally for hooking into job processing frameworks like Sidekiq and Resque.
   attr_accessor :interruption_adapter
+
   self.interruption_adapter = -> { false }
 
   # Set if you want to use your own enumerator builder instead of default EnumeratorBuilder.
@@ -33,21 +34,20 @@ module JobIteration
   #
   #   JobIteration.enumerator_builder = MyOwnBuilder
   attr_accessor :enumerator_builder
+
   self.enumerator_builder = JobIteration::EnumeratorBuilder
 
   def load_integrations
     loaded = nil
     INTEGRATIONS.each do |integration|
-      begin
-        load_integration(integration)
-        if loaded
-          raise IntegrationLoadError,
-            "#{loaded} integration has already been loaded, but #{integration} is also available. " \
+      load_integration(integration)
+      if loaded
+        raise IntegrationLoadError,
+          "#{loaded} integration has already been loaded, but #{integration} is also available. " \
             "Iteration will only work with one integration."
-        end
-        loaded = integration
-      rescue LoadError
       end
+      loaded = integration
+    rescue LoadError
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.1.13
+  version: 1.3.1
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-05-20 00:00:00.000000000 Z
+date: 2021-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -45,6 +45,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
 - ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rubocop.yml"
@@ -52,10 +53,12 @@ files:
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/setup
+- bin/test
 - dev.yml
 - gemfiles/rails_5_2.gemfile
 - gemfiles/rails_6_0.gemfile
@@ -92,14 +95,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.17
+rubygems_version: 3.2.20
 signing_key: 
 specification_version: 4
 summary: Makes your background jobs interruptible and resumable.