job-iteration 1.1.6 → 1.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1aa0774feb655e564e01134b940375a88b52eed719e9ff07bfbadb16c4515b7e
-  data.tar.gz: 54a0dd49a916affafc03122db00a2adbc0069268854fae00b751a35db92ebad4
+  metadata.gz: 18324439fe7b98c7f1bca543c078ddbfdff18af5a7e97a87b21b48890e919769
+  data.tar.gz: 3d912ea06a5a66ee841fbd605e8dfc414bd1760c61d9edf74445e79ae16a6466
 SHA512:
-  metadata.gz: 0d1c6b08ef33f876c8fa692ce655f74502758eb6c0e3a98253d9cbb8a29c67906af2a3cbc317dc103a948c514b9eccad9e5938abc2e803f7f3aa25e4a49e9807
-  data.tar.gz: 3693719f3764bf8e158c568a7b63ed8c1b28174016b3d69fc5743711de99f4ad7bb3e80e29f9851e504e50b52ec1783aade73a7a7ee48b03718a9463aa11e98f
+  metadata.gz: 622368d1208ea23014188c028f832f51f899b6bdfbca8938c6396dcca48f913679ee7f41a035c5cbe0376346ea40a583c4615bd665251cf7dc36a30aeb37904a
+  data.tar.gz: 51931d565cffb4600141e1e96dc9a1a6b8a522589c8b0b38dc4cd1846e8d59a9f150d0d0bfcde4aaee587478af57418093a9e313b2f17c9f2c54ffb6cf18382a

data/.github/workflows/ci.yml

@@ -0,0 +1,47 @@
+name: CI
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} | Gemfile ${{ matrix.gemfile }}
+    services:
+      redis:
+        image: redis
+        ports:
+        - 6379:6379
+    strategy:
+      matrix:
+        ruby: [2.5, 2.6, 2.7, 3.0]
+        gemfile: [rails_5_2, rails_6_0, rails_edge]
+        exclude:
+          - ruby: 2.5
+            gemfile: rails_edge
+          - ruby: 2.6
+            gemfile: rails_edge
+          - ruby: 3.0
+            gemfile: rails_5_2
+    env:
+      BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+    - name: Set up Ruby ${{ matrix.ruby }}
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+    - name: Start MySQL and create DB
+      run: |
+        sudo systemctl start mysql.service
+        mysql -uroot -h localhost -proot -e "CREATE DATABASE job_iteration_test;"
+    - name: Rubocop
+      run: bundle exec rubocop
+    - name: Ruby tests
+      run: bundle exec rake test
+      env:
+        REDIS_HOST: localhost
+        REDIS_PORT: ${{ job.services.redis.ports[6379] }}
+    - name: Documentation correctly written
+      run: bundle exec yardoc --no-output --no-save --no-stats --fail-on-warning

data/.rubocop.yml

@@ -1,5 +1,5 @@
-inherit_from:
-  - http://shopify.github.io/ruby-style-guide/rubocop.yml
+inherit_gem:
+  rubocop-shopify: rubocop.yml
 
 AllCops:
   TargetRubyVersion: 2.4.4

data/CHANGELOG.md

@@ -4,6 +4,29 @@
 
 #### Bug fix
 
+
+## v1.1.11 (April 19, 2021)
+
+#### Bug fix
+
+- [73](https://github.com/Shopify/job-iteration/pull/73) - Enforce cursor be serializable
+
+## v1.1.10 (March 30, 2021)
+
+- [69](https://github.com/Shopify/job-iteration/pull/69) - Fix memory leak in ActiveRecordCursor
+
+## v1.1.9 (January 6, 2021)
+
+- [61](https://github.com/Shopify/job-iteration/pull/61) - Call `super` in `method_added`
+
+## v1.1.8 (June 8, 2020)
+
+- Preserve ruby2_keywords tags in arguments on Ruby 2.7
+
+## v1.1.7 (June 4, 2020)
+
+- [54](https://github.com/Shopify/job-iteration/pull/54) - Fix warnings on Ruby 2.7
+
 ## v1.1.6 (May 22, 2020)
 
 - [49](https://github.com/Shopify/job-iteration/pull/49) -  Log when enumerator has nothing to iterate

data/Gemfile

@@ -8,21 +8,21 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 gemspec
 
 # for integration testing
-gem 'sidekiq'
-gem 'resque'
+gem "sidekiq"
+gem "resque"
 
-gem 'mysql2', '~> 0.5'
-gem 'globalid'
-gem 'i18n'
-gem 'redis'
-gem 'database_cleaner'
+gem "mysql2", "~> 0.5"
+gem "globalid"
+gem "i18n"
+gem "redis"
+gem "database_cleaner"
 
-gem 'pry'
-gem 'mocha'
+gem "pry"
+gem "mocha"
 
-gem 'rubocop', '~> 0.77.0'
-gem 'yard'
-gem 'rake'
+gem "rubocop-shopify", require: false
+gem "yard"
+gem "rake"
 
 # for unit testing optional sorbet support
-gem 'sorbet-runtime'
+gem "sorbet-runtime"

data/README.md

@@ -9,7 +9,7 @@ Meet Iteration, an extension for [ActiveJob](https://github.com/rails/rails/tree
 Imagine the following job:
 
 ```ruby
-class SimpleJob < ActiveJob::Base
+class SimpleJob < ApplicationJob
   def perform
     User.find_each do |user|
       user.notify_about_something
@@ -43,7 +43,7 @@ And then execute:
 In the job, include `JobIteration::Iteration` module and start describing the job with two methods (`build_enumerator` and `each_iteration`) instead of `perform`:
 
 ```ruby
-class NotifyUsersJob < ActiveJob::Base
+class NotifyUsersJob < ApplicationJob
   include JobIteration::Iteration
 
   def build_enumerator(cursor:)
@@ -64,7 +64,9 @@ end
 Check out more examples of Iterations:
 
 ```ruby
-class BatchesJob < ActiveJob::Iteration
+class BatchesJob < ApplicationJob
+  include JobIteration::Iteration
+
   def build_enumerator(product_id, cursor:)
     enumerator_builder.active_record_on_batches(
       Product.find(product_id).comments,
@@ -81,7 +83,9 @@ end
 ```
 
 ```ruby
-class ArrayJob < ActiveJob::Iteration
+class ArrayJob < ApplicationJob
+  include JobIteration::Iteration
+
   def build_enumerator(cursor:)
     enumerator_builder.array(['build', 'enumerator', 'from', 'any', 'array'], cursor: cursor)
   end
@@ -93,7 +97,9 @@ end
 ```
 
 ```ruby
-class CsvJob < ActiveJob::Iteration
+class CsvJob < ApplicationJob
+  include JobIteration::Iteration
+
   def build_enumerator(import_id, cursor:)
     import = Import.find(import_id)
     JobIteration::CsvEnumerator.new(import.csv).rows(cursor: cursor)
@@ -153,7 +159,7 @@ There a few configuration assumptions that are required for Iteration to work wi
 **My job has a complex flow. How do I write my own Enumerator?** Iteration API takes care of persisting the cursor (that you may use to calculate an offset) and controlling the job state. The power of Enumerator object is that you can use the cursor in any way you want. One example is a cursorless job that pops records from a datastore until the job is interrupted:
 
 ```ruby
-class MyJob < ActiveJob::Base
+class MyJob < ApplicationJob
   include JobIteration::Iteration
 
   def build_enumerator(cursor:)

data/dev.yml

@@ -7,7 +7,7 @@ up:
     - mysql-client:
         or:        [mysql@5.7]
   - ruby:
-      version: 2.6.2
+      version: 2.6.5
   - railgun
   - bundler
   - custom:

data/gemfiles/rails_5_2.gemfile

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-eval_gemfile '../Gemfile'
+eval_gemfile "../Gemfile"
 
-gem 'activejob', '~> 5.2.0'
-gem 'activerecord', '~> 5.2.0'
+gem "activejob", "~> 5.2.0"
+gem "activerecord", "~> 5.2.0"

data/gemfiles/rails_6_0.gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+eval_gemfile "../Gemfile"
+
+gem "activejob", "~> 6.0.0"
+gem "activerecord", "~> 6.0.0"

data/gemfiles/rails_edge.gemfile

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-eval_gemfile '../Gemfile'
+eval_gemfile "../Gemfile"
 
-gem 'activejob', github: 'rails/rails'
-gem 'activerecord', github: 'rails/rails'
+gem "activejob", github: "rails/rails", branch: "main"
+gem "activerecord", github: "rails/rails", branch: "main"

data/guides/custom-enumerator.md

@@ -72,3 +72,5 @@ end
 ```
 
 We recommend that you read the implementation of the other enumerators that come with the library (`CsvEnumerator`, `ActiveRecordEnumerator`) to gain a better understanding of building Enumerator objects.
+
+Code that is written after the `yield` in a custom enumerator is not guaranteed to execute. In the case that a job is forced to exit ie `job_should_exit?` is true, then the job is re-enqueued during the yield and the rest of the code in the enumerator does not run. You can follow that logic [here](https://github.com/Shopify/job-iteration/blob/9641f455b9126efff2214692c0bef423e0d12c39/lib/job-iteration/iteration.rb#L128-L131).

data/job-iteration.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.authors       = %w(Shopify)
   spec.email         = ["ops-accounts+shipit@shopify.com"]
 
-  spec.summary       = 'Makes your background jobs interruptible and resumable.'
+  spec.summary       = "Makes your background jobs interruptible and resumable."
   spec.description   = spec.summary
   spec.homepage      = "https://github.com/shopify/job-iteration"
   spec.license       = "MIT"

data/lib/job-iteration.rb

@@ -54,11 +54,11 @@ module JobIteration
   def load_integration(integration)
     unless INTEGRATIONS.include?(integration)
       raise IntegrationLoadError,
-        "#{integration} integration is not supported. Available integrations: #{INTEGRATIONS.join(', ')}"
+        "#{integration} integration is not supported. Available integrations: #{INTEGRATIONS.join(", ")}"
     end
 
     require_relative "./job-iteration/integrations/#{integration}"
   end
 end
 
-JobIteration.load_integrations unless ENV['ITERATION_DISABLE_AUTOCONFIGURE']
+JobIteration.load_integrations unless ENV["ITERATION_DISABLE_AUTOCONFIGURE"]

data/lib/job-iteration/active_record_cursor.rb

@@ -23,7 +23,7 @@ module JobIteration
       @columns = Array.wrap(columns)
       self.position = Array.wrap(position)
       raise ArgumentError, "Must specify at least one column" if columns.empty?
-      if relation.joins_values.present? && !@columns.all? { |column| column.to_s.include?('.') }
+      if relation.joins_values.present? && !@columns.all? { |column| column.to_s.include?(".") }
         raise ArgumentError, "You need to specify fully-qualified columns if you join a table"
       end
 
@@ -31,7 +31,7 @@ module JobIteration
         raise ConditionNotSupportedError
       end
 
-      @base_relation = relation.reorder(@columns.join(','))
+      @base_relation = relation.reorder(@columns.join(","))
       @reached_end = false
     end
 
@@ -50,7 +50,7 @@ module JobIteration
 
     def update_from_record(record)
       self.position = @columns.map do |column|
-        method = column.to_s.split('.').last
+        method = column.to_s.split(".").last
         record.send(method.to_sym)
       end
     end
@@ -64,7 +64,9 @@ module JobIteration
         relation = relation.where(*conditions)
       end
 
-      records = relation.to_a
+      records = relation.uncached do
+        relation.to_a
+      end
 
       update_from_record(records.last) unless records.empty?
       @reached_end = records.size < batch_size

data/lib/job-iteration/active_record_enumerator.rb

@@ -40,7 +40,7 @@ module JobIteration
 
     def cursor_value(record)
       positions = @columns.map do |column|
-        attribute_name = column.to_s.split('.').last
+        attribute_name = column.to_s.split(".").last
         column_value(record, attribute_name)
       end
       return positions.first if positions.size == 1

data/lib/job-iteration/integrations/resque.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'resque'
+require "resque"
 
 module JobIteration
   module Integrations

data/lib/job-iteration/integrations/sidekiq.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'sidekiq'
+require "sidekiq"
 
 module JobIteration
   module Integrations # @private

data/lib/job-iteration/iteration.rb

@@ -1,11 +1,33 @@
 # frozen_string_literal: true
 
-require 'active_support/all'
+require "active_support/all"
 
 module JobIteration
   module Iteration
     extend ActiveSupport::Concern
 
+    class CursorError < ArgumentError
+      attr_reader :cursor
+
+      def initialize(message, cursor:)
+        super(message)
+        @cursor = cursor
+      end
+
+      def message
+        "#{super} (#{inspected_cursor})"
+      end
+
+      private
+
+      def inspected_cursor
+        cursor.inspect
+      rescue NoMethodError
+        # For those brave enough to try to use BasicObject as cursor. Nice try.
+        Object.instance_method(:inspect).bind(cursor).call
+      end
+    end
+
     included do |_base|
       attr_accessor(
         :cursor_position,
@@ -22,6 +44,7 @@ module JobIteration
     module ClassMethods
       def method_added(method_name)
         ban_perform_definition if method_name.to_sym == :perform
+        super
       end
 
       def on_start(*filters, &blk)
@@ -49,27 +72,28 @@ module JobIteration
       self.total_time = 0.0
       assert_implements_methods!
     end
+    ruby2_keywords(:initialize) if respond_to?(:ruby2_keywords, true)
 
     def serialize # @private
       super.merge(
-        'cursor_position' => cursor_position,
-        'times_interrupted' => times_interrupted,
-        'total_time' => total_time,
+        "cursor_position" => cursor_position,
+        "times_interrupted" => times_interrupted,
+        "total_time" => total_time,
       )
     end
 
     def deserialize(job_data) # @private
       super
-      self.cursor_position = job_data['cursor_position']
-      self.times_interrupted = job_data['times_interrupted'] || 0
-      self.total_time = job_data['total_time'] || 0
+      self.cursor_position = job_data["cursor_position"]
+      self.times_interrupted = job_data["times_interrupted"] || 0
+      self.total_time = job_data["total_time"] || 0
     end
 
     def perform(*params) # @private
       interruptible_perform(*params)
     end
 
-    def retry_job(*)
+    def retry_job(*, **)
       super unless defined?(@retried) && @retried
       @retried = true
     end
@@ -118,6 +142,8 @@ module JobIteration
       arguments = arguments.dup.freeze
       found_record = false
       enumerator.each do |object_from_enumerator, index|
+        assert_valid_cursor!(index)
+
         record_unit_of_work do
           found_record = true
           each_iteration(object_from_enumerator, *arguments)
@@ -174,6 +200,18 @@ module JobIteration
       EOS
     end
 
+    # The adapter must be able to serialize and deserialize the cursor back into an equivalent object.
+    # https://github.com/mperham/sidekiq/wiki/Best-Practices#1-make-your-job-parameters-small-and-simple
+    def assert_valid_cursor!(cursor)
+      return if serializable?(cursor)
+
+      raise CursorError.new(
+        "Cursor must be composed of objects capable of built-in (de)serialization: " \
+        "Strings, Integers, Floats, Arrays, Hashes, true, false, or nil.",
+        cursor: cursor,
+      )
+    end
+
     def assert_implements_methods!
       unless respond_to?(:each_iteration, true)
         raise(
@@ -249,5 +287,21 @@ module JobIteration
       end
       false
     end
+
+    SIMPLE_SERIALIZABLE_CLASSES = [String, Integer, Float, NilClass, TrueClass, FalseClass].freeze
+    private_constant :SIMPLE_SERIALIZABLE_CLASSES
+    def serializable?(object)
+      # Subclasses must be excluded, hence not using is_a? or ===.
+      if object.instance_of?(Array)
+        object.all? { |element| serializable?(element) }
+      elsif object.instance_of?(Hash)
+        object.all? { |key, value| serializable?(key) && serializable?(value) }
+      else
+        SIMPLE_SERIALIZABLE_CLASSES.any? { |klass| object.instance_of?(klass) }
+      end
+    rescue NoMethodError
+      # BasicObject doesn't respond to instance_of, but we can't serialize it anyway
+      false
+    end
   end
 end

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.1.6"
+  VERSION = "1.1.11"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.1.6
+  version: 1.1.11
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-22 00:00:00.000000000 Z
+date: 2021-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -45,9 +45,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
 - ".yardopts"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -58,6 +58,7 @@ files:
 - bin/setup
 - dev.yml
 - gemfiles/rails_5_2.gemfile
+- gemfiles/rails_6_0.gemfile
 - gemfiles/rails_edge.gemfile
 - guides/best-practices.md
 - guides/custom-enumerator.md

data/.travis.yml

@@ -1,17 +0,0 @@
-services:
-  - mysql
-  - redis-server
-language: ruby
-rvm:
-  - 2.5.5
-  - 2.6.2
-before_install:
-  - mysql -e 'CREATE DATABASE job_iteration_test;'
-script:
-  - bundle exec rake test
-  - bundle exec rubocop
-  - bundle exec yardoc --no-output --no-save --no-stats --fail-on-warning
-
-gemfile:
-  - 'gemfiles/rails_5_2.gemfile'
-  - 'gemfiles/rails_edge.gemfile'