job-iteration 1.1.9 → 1.1.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1a4821d55797f7597b3c916d6fbca2195c054bd3aab0ffcc40c9ce21ae0b1ec4
-  data.tar.gz: e71ce0fb7ec744b502f018ccfc7ea5e819bfe06f9fb2ca5a32c70cc1b875abc7
+  metadata.gz: 0da9051861fb27696febf2d0191b4d2f459ece64b34cf0231398984c0f36ef3d
+  data.tar.gz: 5fc84c784cdc1a0558a96891a236ac3cea315068e32a6730d633427ef584204b
 SHA512:
-  metadata.gz: 943795fabf60cff76fa7b8678f1f5d5a83bd98340194e142092dc891ff1abe991c0410572cebd09155c6c220ff96aa5530fb27e1b4fe119c6bbbf22fe12670ef
-  data.tar.gz: 11ea6f0165aa31350c6486236a33d66c94316a08f52905aeaab049388b433c229ccf52e69ae3e43eb8bca9a5c56dc4aae46803aae4add9d2a052d5d49af14d8d
+  metadata.gz: f30dbb37ada6149fdd877c4b92fff5cf91e078202408230cbdd19593a341c25f4311b4d12dcdceb539a1553e30e0386930a0b49e4c082a4261a0e788c82125b6
+  data.tar.gz: 88e9846617f7c45959b45328e46b1465b5acb43caf1a5e0b9fa4a5d0848d00d842392ef4383e372f68c7ccb7aa3f0954549a64846cd3e15c712d9a0d7a60d781

data/.github/workflows/ci.yml

@@ -0,0 +1,47 @@
+name: CI
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} | Gemfile ${{ matrix.gemfile }}
+    services:
+      redis:
+        image: redis
+        ports:
+        - 6379:6379
+    strategy:
+      matrix:
+        ruby: [2.5, 2.6, 2.7, 3.0]
+        gemfile: [rails_5_2, rails_6_0, rails_edge]
+        exclude:
+          - ruby: 2.5
+            gemfile: rails_edge
+          - ruby: 2.6
+            gemfile: rails_edge
+          - ruby: 3.0
+            gemfile: rails_5_2
+    env:
+      BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v2
+    - name: Set up Ruby ${{ matrix.ruby }}
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+    - name: Start MySQL and create DB
+      run: |
+        sudo systemctl start mysql.service
+        mysql -uroot -h localhost -proot -e "CREATE DATABASE job_iteration_test;"
+    - name: Rubocop
+      run: bundle exec rubocop
+    - name: Ruby tests
+      run: bundle exec rake test
+      env:
+        REDIS_HOST: localhost
+        REDIS_PORT: ${{ job.services.redis.ports[6379] }}
+    - name: Documentation correctly written
+      run: bundle exec yardoc --no-output --no-save --no-stats --fail-on-warning

data/.rubocop.yml

@@ -1,5 +1,5 @@
-inherit_from:
-  - http://shopify.github.io/ruby-style-guide/rubocop.yml
+inherit_gem:
+  rubocop-shopify: rubocop.yml
 
 AllCops:
   TargetRubyVersion: 2.4.4

data/CHANGELOG.md

@@ -1,12 +1,38 @@
 ### Master (unreleased)
 
+## v1.1.14 (May 28, 2021)
+
+#### Bug fix
+- [84](https://github.com/Shopify/job-iteration/pull/84) - Call adjust_total_time before running on_complete callbacks
+- [94](https://github.com/Shopify/job-iteration/pull/94) - Remove unnecessary break
+- [95](https://github.com/Shopify/job-iteration/pull/95) - ActiveRecordBatchEnumerator#each should rewind at the end
+- [97](https://github.com/Shopify/job-iteration/pull/97) - Batch enumerator size returns the number of batches, not records
+
+## v1.1.13 (May 20, 2021)
+
 #### New feature
+- [91](https://github.com/Shopify/job-iteration/pull/91) - Add enumerator yielding batches as Active Record Relations
+
+## v1.1.12 (April 19, 2021)
+
+#### Bug fix
+
+- [77](https://github.com/Shopify/job-iteration/pull/77) - Defer enforce cursor be serializable until 2.0.0
+
+## v1.1.11 (April 19, 2021)
 
 #### Bug fix
 
+- [73](https://github.com/Shopify/job-iteration/pull/73) - Enforce cursor be serializable
+  _This is reverted in 1.1.12 as it breaks behaviour in some apps._
+
+## v1.1.10 (March 30, 2021)
+
+- [69](https://github.com/Shopify/job-iteration/pull/69) - Fix memory leak in ActiveRecordCursor
+
 ## v1.1.9 (January 6, 2021)
 
-- [61](https://github.com/Shopify/job-iteration/pull/61) Call `super` in `method_added`
+- [61](https://github.com/Shopify/job-iteration/pull/61) - Call `super` in `method_added`
 
 ## v1.1.8 (June 8, 2020)
 

data/Gemfile

@@ -8,21 +8,22 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 gemspec
 
 # for integration testing
-gem 'sidekiq'
-gem 'resque'
+gem "sidekiq"
+gem "resque"
 
-gem 'mysql2', '~> 0.5'
-gem 'globalid'
-gem 'i18n'
-gem 'redis'
-gem 'database_cleaner'
+gem "mysql2", "~> 0.5"
+gem "globalid"
+gem "i18n"
+gem "redis"
+gem "database_cleaner"
 
-gem 'pry'
-gem 'mocha'
+gem "pry"
+gem "mocha"
 
-gem 'rubocop', '~> 0.77.0'
-gem 'yard'
-gem 'rake'
+gem "rubocop-shopify", require: false
+gem "rubocop", "<= 1.12.1", require: false # 1.13.0 drops Ruby 2.4 support
+gem "yard"
+gem "rake"
 
 # for unit testing optional sorbet support
-gem 'sorbet-runtime'
+gem "sorbet-runtime"

data/README.md

@@ -77,7 +77,28 @@ class BatchesJob < ApplicationJob
 
   def each_iteration(batch_of_comments, product_id)
     # batch_of_comments will contain batches of 100 records
-    Comment.where(id: batch_of_comments.map(&:id)).update_all(deleted: true)
+    batch_of_comments.each do |comment|
+      DeleteCommentJob.perform_later(comment)
+    end
+  end
+end
+```
+
+```ruby
+class BatchesAsRelationJob < ApplicationJob
+  include JobIteration::Iteration
+
+  def build_enumerator(product_id, cursor:)
+    enumerator_builder.active_record_on_batch_relations(
+      Product.find(product_id).comments,
+      cursor: cursor,
+      batch_size: 100,
+    )
+  end
+
+  def each_iteration(batch_of_comments, product_id)
+    # batch_of_comments will be a Comment::ActiveRecord_Relation
+    batch_of_comments.update_all(deleted: true)
   end
 end
 ```

data/gemfiles/rails_5_2.gemfile

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-eval_gemfile '../Gemfile'
+eval_gemfile "../Gemfile"
 
-gem 'activejob', '~> 5.2.0'
-gem 'activerecord', '~> 5.2.0'
+gem "activejob", "~> 5.2.0"
+gem "activerecord", "~> 5.2.0"

data/gemfiles/rails_6_0.gemfile

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-eval_gemfile '../Gemfile'
+eval_gemfile "../Gemfile"
 
-gem 'activejob', '~> 6.0.0'
-gem 'activerecord', '~> 6.0.0'
+gem "activejob", "~> 6.0.0"
+gem "activerecord", "~> 6.0.0"

data/gemfiles/rails_edge.gemfile

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-eval_gemfile '../Gemfile'
+eval_gemfile "../Gemfile"
 
-gem 'activejob', github: 'rails/rails'
-gem 'activerecord', github: 'rails/rails'
+gem "activejob", github: "rails/rails", branch: "main"
+gem "activerecord", github: "rails/rails", branch: "main"

data/job-iteration.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.authors       = %w(Shopify)
   spec.email         = ["ops-accounts+shipit@shopify.com"]
 
-  spec.summary       = 'Makes your background jobs interruptible and resumable.'
+  spec.summary       = "Makes your background jobs interruptible and resumable."
   spec.description   = spec.summary
   spec.homepage      = "https://github.com/shopify/job-iteration"
   spec.license       = "MIT"

data/lib/job-iteration.rb

@@ -54,11 +54,11 @@ module JobIteration
   def load_integration(integration)
     unless INTEGRATIONS.include?(integration)
       raise IntegrationLoadError,
-        "#{integration} integration is not supported. Available integrations: #{INTEGRATIONS.join(', ')}"
+        "#{integration} integration is not supported. Available integrations: #{INTEGRATIONS.join(", ")}"
     end
 
     require_relative "./job-iteration/integrations/#{integration}"
   end
 end
 
-JobIteration.load_integrations unless ENV['ITERATION_DISABLE_AUTOCONFIGURE']
+JobIteration.load_integrations unless ENV["ITERATION_DISABLE_AUTOCONFIGURE"]

data/lib/job-iteration/active_record_batch_enumerator.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module JobIteration
+  # Builds Batch Enumerator based on ActiveRecord Relation.
+  # @see EnumeratorBuilder
+  class ActiveRecordBatchEnumerator
+    include Enumerable
+
+    SQL_DATETIME_WITH_NSEC = "%Y-%m-%d %H:%M:%S.%N"
+
+    def initialize(relation, columns: nil, batch_size: 100, cursor: nil)
+      @batch_size = batch_size
+      @primary_key = "#{relation.table_name}.#{relation.primary_key}"
+      @columns = Array(columns&.map(&:to_s) || @primary_key)
+      @primary_key_index = @columns.index(@primary_key) || @columns.index(relation.primary_key)
+      @pluck_columns = if @primary_key_index
+        @columns
+      else
+        @columns.dup << @primary_key
+      end
+      @cursor = Array.wrap(cursor)
+      @initial_cursor = @cursor
+      raise ArgumentError, "Must specify at least one column" if @columns.empty?
+      if relation.joins_values.present? && !@columns.all? { |column| column.to_s.include?(".") }
+        raise ArgumentError, "You need to specify fully-qualified columns if you join a table"
+      end
+
+      if relation.arel.orders.present? || relation.arel.taken.present?
+        raise ConditionNotSupportedError
+      end
+
+      @base_relation = relation.reorder(@columns.join(","))
+    end
+
+    def each
+      return to_enum { size } unless block_given?
+      while (relation = next_batch)
+        yield relation, cursor_value
+      end
+    end
+
+    def size
+      (@base_relation.count + @batch_size - 1) / @batch_size # ceiling division
+    end
+
+    private
+
+    def next_batch
+      relation = @base_relation.limit(@batch_size)
+      if conditions.any?
+        relation = relation.where(*conditions)
+      end
+
+      cursor_values, ids = relation.uncached do
+        pluck_columns(relation)
+      end
+
+      cursor = cursor_values.last
+      unless cursor.present?
+        @cursor = @initial_cursor
+        return
+      end
+      # The primary key was plucked, but original cursor did not include it, so we should remove it
+      cursor.pop unless @primary_key_index
+      @cursor = Array.wrap(cursor)
+
+      # Yields relations by selecting the primary keys of records in the batch.
+      # Post.where(published: nil) results in an enumerator of relations like: Post.where(ids: batch_of_ids)
+      @base_relation.where(@primary_key => ids)
+    end
+
+    def pluck_columns(relation)
+      if @pluck_columns.size == 1 # only the primary key
+        column_values = relation.pluck(*@pluck_columns)
+        return [column_values, column_values]
+      end
+
+      column_values = relation.pluck(*@pluck_columns)
+      primary_key_index = @primary_key_index || -1
+      primary_key_values = column_values.map { |values| values[primary_key_index] }
+
+      serialize_column_values!(column_values)
+      [column_values, primary_key_values]
+    end
+
+    def cursor_value
+      return @cursor.first if @cursor.size == 1
+      @cursor
+    end
+
+    def conditions
+      column_index = @cursor.size - 1
+      column = @columns[column_index]
+      where_clause = if @columns.size == @cursor.size
+        "#{column} > ?"
+      else
+        "#{column} >= ?"
+      end
+      while column_index > 0
+        column_index -= 1
+        column = @columns[column_index]
+        where_clause = "#{column} > ? OR (#{column} = ? AND (#{where_clause}))"
+      end
+      ret = @cursor.reduce([where_clause]) { |params, value| params << value << value }
+      ret.pop
+      ret
+    end
+
+    def serialize_column_values!(column_values)
+      column_values.map! { |values| values.map! { |value| column_value(value) } }
+    end
+
+    def column_value(value)
+      value.is_a?(Time) ? value.strftime(SQL_DATETIME_WITH_NSEC) : value
+    end
+  end
+end

data/lib/job-iteration/active_record_cursor.rb

@@ -23,7 +23,7 @@ module JobIteration
       @columns = Array.wrap(columns)
       self.position = Array.wrap(position)
       raise ArgumentError, "Must specify at least one column" if columns.empty?
-      if relation.joins_values.present? && !@columns.all? { |column| column.to_s.include?('.') }
+      if relation.joins_values.present? && !@columns.all? { |column| column.to_s.include?(".") }
         raise ArgumentError, "You need to specify fully-qualified columns if you join a table"
       end
 
@@ -31,7 +31,7 @@ module JobIteration
         raise ConditionNotSupportedError
       end
 
-      @base_relation = relation.reorder(@columns.join(','))
+      @base_relation = relation.reorder(@columns.join(","))
       @reached_end = false
     end
 
@@ -50,7 +50,7 @@ module JobIteration
 
     def update_from_record(record)
       self.position = @columns.map do |column|
-        method = column.to_s.split('.').last
+        method = column.to_s.split(".").last
         record.send(method.to_sym)
       end
     end
@@ -64,7 +64,9 @@ module JobIteration
         relation = relation.where(*conditions)
       end
 
-      records = relation.to_a
+      records = relation.uncached do
+        relation.to_a
+      end
 
       update_from_record(records.last) unless records.empty?
       @reached_end = records.size < batch_size

data/lib/job-iteration/active_record_enumerator.rb

@@ -40,7 +40,7 @@ module JobIteration
 
     def cursor_value(record)
       positions = @columns.map do |column|
-        attribute_name = column.to_s.split('.').last
+        attribute_name = column.to_s.split(".").last
         column_value(record, attribute_name)
       end
       return positions.first if positions.size == 1

data/lib/job-iteration/enumerator_builder.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require_relative "./active_record_batch_enumerator"
 require_relative "./active_record_enumerator"
 require_relative "./csv_enumerator"
 require_relative "./throttle_enumerator"
@@ -86,6 +87,11 @@ module JobIteration
     #        WHERE (created_at > '$LAST_CREATED_AT_CURSOR'
     #          OR (created_at = '$LAST_CREATED_AT_CURSOR' AND (id > '$LAST_ID_CURSOR')))
     #        ORDER BY created_at, id LIMIT 100
+    #
+    # As a result of this query pattern, if the values in these columns change for the records in scope during
+    # iteration, they may be skipped or yielded multiple times depending on the nature of the update and the
+    # cursor's value. If the value gets updated to a greater value than the cursor's value, it will get yielded
+    # again. Similarly, if the value gets updated to a lesser value than the curor's value, it will get skipped.
     def build_active_record_enumerator_on_records(scope, cursor:, **args)
       enum = build_active_record_enumerator(
         scope,
@@ -95,7 +101,7 @@ module JobIteration
       wrap(self, enum)
     end
 
-    # Builds Enumerator from Active Record Relation and enumerates on batches.
+    # Builds Enumerator from Active Record Relation and enumerates on batches of records.
     # Each Enumerator tick moves the cursor +batch_size+ rows forward.
     #
     # +batch_size:+ sets how many records will be fetched in one batch. Defaults to 100.
@@ -110,6 +116,16 @@ module JobIteration
       wrap(self, enum)
     end
 
+    # Builds Enumerator from Active Record Relation and enumerates on batches, yielding Active Record Relations.
+    # See documentation for #build_active_record_enumerator_on_batches.
+    def build_active_record_enumerator_on_batch_relations(scope, cursor:, **args)
+      JobIteration::ActiveRecordBatchEnumerator.new(
+        scope,
+        cursor: cursor,
+        **args
+      ).each
+    end
+
     def build_throttle_enumerator(enum, throttle_on:, backoff:)
       JobIteration::ThrottleEnumerator.new(
         enum,
@@ -124,6 +140,7 @@ module JobIteration
     alias_method :array, :build_array_enumerator
     alias_method :active_record_on_records, :build_active_record_enumerator_on_records
     alias_method :active_record_on_batches, :build_active_record_enumerator_on_batches
+    alias_method :active_record_on_batch_relations, :build_active_record_enumerator_on_batch_relations
     alias_method :throttle, :build_throttle_enumerator
 
     private

data/lib/job-iteration/integrations/resque.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'resque'
+require "resque"
 
 module JobIteration
   module Integrations

data/lib/job-iteration/integrations/sidekiq.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'sidekiq'
+require "sidekiq"
 
 module JobIteration
   module Integrations # @private

data/lib/job-iteration/iteration.rb

@@ -1,11 +1,33 @@
 # frozen_string_literal: true
 
-require 'active_support/all'
+require "active_support/all"
 
 module JobIteration
   module Iteration
     extend ActiveSupport::Concern
 
+    class CursorError < ArgumentError
+      attr_reader :cursor
+
+      def initialize(message, cursor:)
+        super(message)
+        @cursor = cursor
+      end
+
+      def message
+        "#{super} (#{inspected_cursor})"
+      end
+
+      private
+
+      def inspected_cursor
+        cursor.inspect
+      rescue NoMethodError
+        # For those brave enough to try to use BasicObject as cursor. Nice try.
+        Object.instance_method(:inspect).bind(cursor).call
+      end
+    end
+
     included do |_base|
       attr_accessor(
         :cursor_position,
@@ -54,17 +76,17 @@ module JobIteration
 
     def serialize # @private
       super.merge(
-        'cursor_position' => cursor_position,
-        'times_interrupted' => times_interrupted,
-        'total_time' => total_time,
+        "cursor_position" => cursor_position,
+        "times_interrupted" => times_interrupted,
+        "total_time" => total_time,
       )
     end
 
     def deserialize(job_data) # @private
       super
-      self.cursor_position = job_data['cursor_position']
-      self.times_interrupted = job_data['times_interrupted'] || 0
-      self.total_time = job_data['total_time'] || 0
+      self.cursor_position = job_data["cursor_position"]
+      self.times_interrupted = job_data["times_interrupted"] || 0
+      self.total_time = job_data["total_time"] || 0
     end
 
     def perform(*params) # @private
@@ -120,6 +142,9 @@ module JobIteration
       arguments = arguments.dup.freeze
       found_record = false
       enumerator.each do |object_from_enumerator, index|
+        # Deferred until 2.0.0
+        # assert_valid_cursor!(index)
+
         record_unit_of_work do
           found_record = true
           each_iteration(object_from_enumerator, *arguments)
@@ -137,6 +162,8 @@ module JobIteration
         "times_interrupted=#{times_interrupted} cursor_position=#{cursor_position}"
       ) unless found_record
 
+      adjust_total_time
+
       true
     end
 
@@ -176,6 +203,18 @@ module JobIteration
       EOS
     end
 
+    # The adapter must be able to serialize and deserialize the cursor back into an equivalent object.
+    # https://github.com/mperham/sidekiq/wiki/Best-Practices#1-make-your-job-parameters-small-and-simple
+    def assert_valid_cursor!(cursor)
+      return if serializable?(cursor)
+
+      raise CursorError.new(
+        "Cursor must be composed of objects capable of built-in (de)serialization: " \
+        "Strings, Integers, Floats, Arrays, Hashes, true, false, or nil.",
+        cursor: cursor,
+      )
+    end
+
     def assert_implements_methods!
       unless respond_to?(:each_iteration, true)
         raise(
@@ -212,8 +251,6 @@ module JobIteration
     end
 
     def output_interrupt_summary
-      adjust_total_time
-
       message = "[JobIteration::Iteration] Completed iterating. times_interrupted=%d total_time=%.3f"
       logger.info(Kernel.format(message, times_interrupted, total_time))
     end
@@ -251,5 +288,21 @@ module JobIteration
       end
       false
     end
+
+    SIMPLE_SERIALIZABLE_CLASSES = [String, Integer, Float, NilClass, TrueClass, FalseClass].freeze
+    private_constant :SIMPLE_SERIALIZABLE_CLASSES
+    def serializable?(object)
+      # Subclasses must be excluded, hence not using is_a? or ===.
+      if object.instance_of?(Array)
+        object.all? { |element| serializable?(element) }
+      elsif object.instance_of?(Hash)
+        object.all? { |key, value| serializable?(key) && serializable?(value) }
+      else
+        SIMPLE_SERIALIZABLE_CLASSES.any? { |klass| object.instance_of?(klass) }
+      end
+    rescue NoMethodError
+      # BasicObject doesn't respond to instance_of, but we can't serialize it anyway
+      false
+    end
   end
 end

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.1.9"
+  VERSION = "1.1.14"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.1.9
+  version: 1.1.14
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-01-06 00:00:00.000000000 Z
+date: 2021-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -45,9 +45,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rubocop.yml"
-- ".travis.yml"
 - ".yardopts"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -66,6 +66,7 @@ files:
 - guides/throttling.md
 - job-iteration.gemspec
 - lib/job-iteration.rb
+- lib/job-iteration/active_record_batch_enumerator.rb
 - lib/job-iteration/active_record_cursor.rb
 - lib/job-iteration/active_record_enumerator.rb
 - lib/job-iteration/csv_enumerator.rb
@@ -98,7 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.17
 signing_key: 
 specification_version: 4
 summary: Makes your background jobs interruptible and resumable.

data/.travis.yml

@@ -1,19 +0,0 @@
-services:
-  - mysql
-  - redis-server
-language: ruby
-rvm:
-  - 2.5
-  - 2.6
-  - 2.7
-before_install:
-  - mysql -e 'CREATE DATABASE job_iteration_test;'
-script:
-  - bundle exec rake test
-  - bundle exec rubocop
-  - bundle exec yardoc --no-output --no-save --no-stats --fail-on-warning
-
-gemfile:
-  - 'gemfiles/rails_5_2.gemfile'
-  - 'gemfiles/rails_6_0.gemfile'
-  - 'gemfiles/rails_edge.gemfile'