job-iteration 1.1.14 → 1.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0da9051861fb27696febf2d0191b4d2f459ece64b34cf0231398984c0f36ef3d
-  data.tar.gz: 5fc84c784cdc1a0558a96891a236ac3cea315068e32a6730d633427ef584204b
+  metadata.gz: 63917aaf8f527e78cd47f62111dbda8473c90907eeaa8827ed23aaba77f494a0
+  data.tar.gz: 4fa0d5ce8eb5c1e7cdc2712daae6ddf689cfd1ae7cc3f840abe7b45628ee7735
 SHA512:
-  metadata.gz: f30dbb37ada6149fdd877c4b92fff5cf91e078202408230cbdd19593a341c25f4311b4d12dcdceb539a1553e30e0386930a0b49e4c082a4261a0e788c82125b6
-  data.tar.gz: 88e9846617f7c45959b45328e46b1465b5acb43caf1a5e0b9fa4a5d0848d00d842392ef4383e372f68c7ccb7aa3f0954549a64846cd3e15c712d9a0d7a60d781
+  metadata.gz: ae0a1330b24adfa55c045cd079806c350e151ee7c12555c9397143919c8f77fdc3023cb23330fc529af3771da31379991830513ce0459e1ff5ff21614e0518d0
+  data.tar.gz: 0733a5377c27ff39acd118f3e719e52bd82a7a63d7f03508b61c6ced900baef9bc0e05bdc5b43d87d6e36f05339bf9a84c1ffd5c1ce70c06cb172ee3c3905240

data/.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+
+updates:
+
+  - package-ecosystem: bundler
+    directory: '/'
+    versioning-strategy: increase
+    open-pull-requests-limit: 100
+    insecure-external-code-execution: allow
+    schedule:
+      interval: weekly
+
+  - package-ecosystem: github-actions
+    directory: '/'
+    schedule:
+      interval: daily

data/.github/workflows/ci.yml

@@ -13,11 +13,9 @@ jobs:
         - 6379:6379
     strategy:
       matrix:
-        ruby: [2.5, 2.6, 2.7, 3.0]
+        ruby: [2.6, 2.7, 3.0]
         gemfile: [rails_5_2, rails_6_0, rails_edge]
         exclude:
-          - ruby: 2.5
-            gemfile: rails_edge
           - ruby: 2.6
             gemfile: rails_edge
           - ruby: 3.0

data/.gitignore

@@ -9,4 +9,3 @@
 .ruby-version
 .rubocop-http---shopify-github-io-ruby-style-guide-rubocop-yml
 gemfiles/*.lock
-Gemfile.lock

data/.rubocop.yml

@@ -2,7 +2,7 @@ inherit_gem:
   rubocop-shopify: rubocop.yml
 
 AllCops:
-  TargetRubyVersion: 2.4.4
+  TargetRubyVersion: 2.6.5
   Exclude:
     - 'vendor/bundle/**/*'
 Lint/SuppressedException:

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 ### Master (unreleased)
 
+## v1.3.2 (Nov 12, 2021)
+- [148](https://github.com/Shopify/job-iteration/pull/148) - Revert "Do not evaluate enumerator when throttled", due to backwards incompatibility.
+
+## v1.3.1 (Nov 11, 2021)
+- [87](https://github.com/Shopify/job-iteration/pull/87) - Do not evaluate enumerator when throttled (REVERTED)
+
+
+## v1.3.0 (Oct 7, 2021)
+- [133](https://github.com/Shopify/job-iteration/pull/133) - Moves attributes out of JobIteration::Iteration included block
+
+
+## v1.2.0 (Sept 21, 2021)
+- [107](https://github.com/Shopify/job-iteration/pull/107) - Remove broken links from README
+- [108](https://github.com/Shopify/job-iteration/pull/108) - Drop support for ruby 2.5
+- [110](https://github.com/Shopify/job-iteration/pull/110) - Update rubocop TargetRubyVersion
+
 ## v1.1.14 (May 28, 2021)
 
 #### Bug fix

data/Gemfile

@@ -21,7 +21,6 @@ gem "pry"
 gem "mocha"
 
 gem "rubocop-shopify", require: false
-gem "rubocop", "<= 1.12.1", require: false # 1.13.0 drops Ruby 2.4 support
 gem "yard"
 gem "rake"
 

data/Gemfile.lock

@@ -0,0 +1,123 @@
+PATH
+  remote: .
+  specs:
+    job-iteration (1.3.2)
+      activejob (>= 5.2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activejob (6.1.4.1)
+      activesupport (= 6.1.4.1)
+      globalid (>= 0.3.6)
+    activemodel (6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activerecord (6.1.4.1)
+      activemodel (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activesupport (6.1.4.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    ast (2.4.2)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.9)
+    connection_pool (2.2.5)
+    database_cleaner (2.0.1)
+      database_cleaner-active_record (~> 2.0.0)
+    database_cleaner-active_record (2.0.1)
+      activerecord (>= 5.a)
+      database_cleaner-core (~> 2.0.0)
+    database_cleaner-core (2.0.1)
+    globalid (0.5.2)
+      activesupport (>= 5.0)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    method_source (1.0.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mono_logger (1.1.1)
+    multi_json (1.15.0)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
+    mysql2 (0.5.3)
+    parallel (1.21.0)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rack (2.2.3)
+    rack-protection (2.1.0)
+      rack
+    rainbow (3.0.0)
+    rake (13.0.6)
+    redis (4.5.1)
+    redis-namespace (1.8.1)
+      redis (>= 3.0.4)
+    regexp_parser (2.1.1)
+    resque (2.1.0)
+      mono_logger (~> 1.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.6)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    rexml (3.2.5)
+    rubocop (1.22.1)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.12.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.12.0)
+      parser (>= 3.0.1.1)
+    rubocop-shopify (2.3.0)
+      rubocop (~> 1.22)
+    ruby-progressbar (1.11.0)
+    ruby2_keywords (0.0.5)
+    sidekiq (6.2.2)
+      connection_pool (>= 2.2.2)
+      rack (~> 2.0)
+      redis (>= 4.2.0)
+    sinatra (2.1.0)
+      mustermann (~> 1.0)
+      rack (~> 2.2)
+      rack-protection (= 2.1.0)
+      tilt (~> 2.0)
+    sorbet-runtime (0.5.9219)
+    tilt (2.0.10)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.1.0)
+    vegas (0.1.11)
+      rack (>= 1.0.0)
+    yard (0.9.26)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord
+  database_cleaner
+  globalid
+  i18n
+  job-iteration!
+  mocha
+  mysql2 (~> 0.5)
+  pry
+  rake
+  redis
+  resque
+  rubocop-shopify
+  sidekiq
+  sorbet-runtime
+  yard
+
+BUNDLED WITH
+   2.2.22

data/README.md

@@ -1,6 +1,6 @@
 # Job Iteration API
 
-[![Build Status](https://travis-ci.com/Shopify/job-iteration.svg?branch=master)](https://travis-ci.com/Shopify/job-iteration)
+[![CI](https://github.com/Shopify/job-iteration/actions/workflows/ci.yml/badge.svg)](https://github.com/Shopify/job-iteration/actions/workflows/ci.yml)
 
 Meet Iteration, an extension for [ActiveJob](https://github.com/rails/rails/tree/master/activejob) that makes your jobs interruptible and resumable, saving all progress that the job has made (aka checkpoint for jobs).
 
@@ -171,7 +171,7 @@ There a few configuration assumptions that are required for Iteration to work wi
 
 **Why is it important that `each_iteration` takes less than 30 seconds?** When the job worker is scheduled for restart or shutdown, it gets a notice to finish remaining unit of work. To guarantee that no progress is lost we need to make sure that `each_iteration` completes within a reasonable amount of time.
 
-**What do I do if each iteration takes a long time, because it's doing nested operations?** If your `each_iteration` is complex, we recommend enqueuing another job, which will run your nested business logic. We may expose primitives in the future to do this more effectively, but this is not terribly common today. We recommend to read https://goo.gl/UobaaU to learn more about nested operations.
+**What do I do if each iteration takes a long time, because it's doing nested operations?** If your `each_iteration` is complex, we recommend enqueuing another job, which will run your nested business logic. We may expose primitives in the future to do this more effectively, but this is not terribly common today.
 
 **Why do I use have to use this ugly helper in `build_enumerator`? Why can't you automatically infer it?** This is how the first version of the API worked. We checked the type of object returned by `build_enumerable`, and whether it was ActiveRecord Relation or an Array, we used the matching adapter. This caused opaque type branching in Iteration internals and it didn’t allow developers to craft their own Enumerators and control the cursor value. We made a decision to _always_ return Enumerator instance from `build_enumerator`. Now we provide explicit helpers to convert ActiveRecord Relation or an Array to Enumerator, and for more complex iteration flows developers can build their own `Enumerator` objects.
 

data/bin/test

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+def main
+  begin
+    command = create_command
+  rescue ArgumentError => e
+    abort(e.message)
+  end
+  puts "Running #{command.join(" ")}"
+  system(*command)
+end
+
+def create_command
+  case ARGV.length
+  when 0
+    ["bundle", "exec", "rake", "test"]
+  when 1
+    filename = ARGV[0]
+    ["bundle", "exec", "rake", "test", "TEST=#{filename}"]
+  when 2
+    filename = ARGV[0]
+    test_name = ARGV[1]
+    test_name_with_underscores = test_name.tr(" ", "_")
+    test_name_pattern = "/#{Regexp.escape(test_name_with_underscores)}/"
+    ["bundle", "exec", "rake", "test", "TEST=#{filename}", "TESTOPTS=\"--name=#{test_name_pattern} -v\""]
+  else
+    raise ArgumentError, "Too many arguments. Did you forget to put the test name in quotes?"
+  end
+end
+
+main

data/dev.yml

@@ -13,7 +13,43 @@ up:
   - custom:
       name: Create Job Iteration database
       meet: mysql -uroot -h job-iteration.railgun -e "CREATE DATABASE job_iteration_test"
-      met?: mysql -uroot -h job-iteration.railgun job_iteration_test -e "SELECT 1"
+      met?: mysql -uroot -h job-iteration.railgun job_iteration_test -e "SELECT 1" &> /dev/null
 
 commands:
-  test: bundle exec rake
+  test:
+    run:  bin/test "$@"
+    syntax:
+      optional: filename testnamepattern
+    aliases: [t]
+    desc: run tests
+    long_desc: |
+      {{bold:Default}}
+      =======
+      Run the entire test suite.
+
+        Examples:
+        {{command:dev test}}
+        {{command:dev t}}
+
+      {{bold:Run all tests in a file}}
+      ========================
+      Include the file path.
+
+        Example:
+        {{command:dev test test/unit/iteration_test.rb}}
+
+      {{bold:Run a single test in a given file}}
+      ========================
+      Include the file path and the name of the test you'd like to run.
+
+        Example:
+        {{command:dev test test/unit/iteration_test.rb test_that_it_has_a_version_number}}
+
+      {{bold:Run all tests in a given file whose name contains a string}}
+      ========================
+      Include the file path and the string that the test names should contain.
+
+        Example:
+        {{command:dev test test/unit/iteration_test.rb version_number}}
+  style:
+    run: bundle exec rubocop -a

data/job-iteration.gemspec

@@ -5,9 +5,10 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "job-iteration/version"
 
 Gem::Specification.new do |spec|
+  spec.required_ruby_version = ">= 2.6"
   spec.name          = "job-iteration"
   spec.version       = JobIteration::VERSION
-  spec.authors       = %w(Shopify)
+  spec.authors       = ["Shopify"]
   spec.email         = ["ops-accounts+shipit@shopify.com"]
 
   spec.summary       = "Makes your background jobs interruptible and resumable."
@@ -20,7 +21,7 @@ Gem::Specification.new do |spec|
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = %w(lib)
+  spec.require_paths = ["lib"]
 
   spec.metadata["changelog_uri"] = "https://github.com/Shopify/job-iteration/blob/master/CHANGELOG.md"
   spec.metadata["allowed_push_host"] = "https://rubygems.org"

data/lib/job-iteration/active_record_cursor.rb

@@ -13,7 +13,7 @@ module JobIteration
       def initialize
         super(
           "The relation cannot use ORDER BY or LIMIT due to the way how iteration with a cursor is designed. " \
-          "You can use other ways to limit the number of rows, e.g. a WHERE condition on the primary key column."
+            "You can use other ways to limit the number of rows, e.g. a WHERE condition on the primary key column."
         )
       end
     end

data/lib/job-iteration/active_record_enumerator.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require_relative "./active_record_cursor"
 module JobIteration
   # Builds Enumerator based on ActiveRecord Relation. Supports enumerating on rows and batches.

data/lib/job-iteration/enumerator_builder.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require_relative "./active_record_batch_enumerator"
 require_relative "./active_record_enumerator"
 require_relative "./csv_enumerator"

data/lib/job-iteration/iteration.rb

@@ -6,6 +6,13 @@ module JobIteration
   module Iteration
     extend ActiveSupport::Concern
 
+    attr_accessor(
+      :cursor_position,
+      :start_time,
+      :times_interrupted,
+      :total_time,
+    )
+
     class CursorError < ArgumentError
       attr_reader :cursor
 
@@ -29,13 +36,6 @@ module JobIteration
     end
 
     included do |_base|
-      attr_accessor(
-        :cursor_position,
-        :start_time,
-        :times_interrupted,
-        :total_time,
-      )
-
       define_callbacks :start
       define_callbacks :shutdown
       define_callbacks :complete
@@ -159,7 +159,7 @@ module JobIteration
 
       logger.info(
         "[JobIteration::Iteration] Enumerator found nothing to iterate! " \
-        "times_interrupted=#{times_interrupted} cursor_position=#{cursor_position}"
+          "times_interrupted=#{times_interrupted} cursor_position=#{cursor_position}"
       ) unless found_record
 
       adjust_total_time
@@ -167,10 +167,8 @@ module JobIteration
       true
     end
 
-    def record_unit_of_work
-      ActiveSupport::Notifications.instrument("each_iteration.iteration", iteration_instrumentation_tags) do
-        yield
-      end
+    def record_unit_of_work(&block)
+      ActiveSupport::Notifications.instrument("each_iteration.iteration", iteration_instrumentation_tags, &block)
     end
 
     def reenqueue_iteration_job
@@ -210,7 +208,7 @@ module JobIteration
 
       raise CursorError.new(
         "Cursor must be composed of objects capable of built-in (de)serialization: " \
-        "Strings, Integers, Floats, Arrays, Hashes, true, false, or nil.",
+          "Strings, Integers, Floats, Arrays, Hashes, true, false, or nil.",
         cursor: cursor,
       )
     end
@@ -227,7 +225,7 @@ module JobIteration
         parameters = method_parameters(:build_enumerator)
         unless valid_cursor_parameter?(parameters)
           raise ArgumentError, "Iteration job (#{self.class}) #build_enumerator " \
-          "expects the keyword argument `cursor`"
+            "expects the keyword argument `cursor`"
         end
       else
         raise ArgumentError, "Iteration job (#{self.class}) must implement #build_enumerator " \

data/lib/job-iteration/throttle_enumerator.rb

@@ -1,5 +1,6 @@
 # typed: true
 # frozen_string_literal: true
+
 module JobIteration
   # ThrottleEnumerator allows you to throttle iterations
   # based on external signal (e.g. database health).

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.1.14"
+  VERSION = "1.3.2"
 end

data/lib/job-iteration.rb

@@ -22,6 +22,7 @@ module JobIteration
 
   # Used internally for hooking into job processing frameworks like Sidekiq and Resque.
   attr_accessor :interruption_adapter
+
   self.interruption_adapter = -> { false }
 
   # Set if you want to use your own enumerator builder instead of default EnumeratorBuilder.
@@ -33,21 +34,20 @@ module JobIteration
   #
   #   JobIteration.enumerator_builder = MyOwnBuilder
   attr_accessor :enumerator_builder
+
   self.enumerator_builder = JobIteration::EnumeratorBuilder
 
   def load_integrations
     loaded = nil
     INTEGRATIONS.each do |integration|
-      begin
-        load_integration(integration)
-        if loaded
-          raise IntegrationLoadError,
-            "#{loaded} integration has already been loaded, but #{integration} is also available. " \
+      load_integration(integration)
+      if loaded
+        raise IntegrationLoadError,
+          "#{loaded} integration has already been loaded, but #{integration} is also available. " \
             "Iteration will only work with one integration."
-        end
-        loaded = integration
-      rescue LoadError
       end
+      loaded = integration
+    rescue LoadError
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.1.14
+  version: 1.3.2
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-05-28 00:00:00.000000000 Z
+date: 2021-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -45,6 +45,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
 - ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rubocop.yml"
@@ -52,10 +53,12 @@ files:
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/setup
+- bin/test
 - dev.yml
 - gemfiles/rails_5_2.gemfile
 - gemfiles/rails_6_0.gemfile
@@ -92,14 +95,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.17
+rubygems_version: 3.2.20
 signing_key: 
 specification_version: 4
 summary: Makes your background jobs interruptible and resumable.