job-iteration 1.1.12 → 1.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2109f33c06a85e1b336876ea4c0aafe2dc0ea7f17fbdda0d4bba06177f940113
-  data.tar.gz: 3ce3b8017f1dfaac8bb3c83f8b846c4068ed09a0968f937bb1bd3e31d9d1341a
+  metadata.gz: c1707731e6bf03617bdbaa60bc2ddeb9652f34534a8fe0d69b26e233c790d469
+  data.tar.gz: a981f1230a7784e4e6673c0d2e4c0642f4a1d3c48baeb3039ca6e41daced9107
 SHA512:
-  metadata.gz: d22611524765adb75a6d6cd457d2c1e0e7169621b3c9a2fd4f8fbcc4d68f7a79cf89764852a61f11bdd586ac60c1bb9d6c76f34426c5a1eb25299bd8b6ec77b7
-  data.tar.gz: 952569f76f1d9b739d4219bfb3df0878ef1ec1c3784bba985f2246a956a581c51723e113ca10ac9eec3f4c2ae61c6acaabd4d461aa321c1af989d1921293ef34
+  metadata.gz: db396697d25d4b0450b69d2a58d68e1e247a074f02c9a5887746184789f2aca2569e345ef0d88fb5e142fcaa1c0586169ef91b7bc0ffe2ef35796f544e0cafa1
+  data.tar.gz: 757b0c074830cbe97d49fd6e84c59ade45c4b5ecdc57b52cf17ea5539e9f2bbffb8ea929804da531c217bf43bbc4f67f726e9d0f1a9a391e32e35837b7edf624

data/CHANGELOG.md

@@ -1,8 +1,9 @@
 ### Master (unreleased)
 
-#### New feature
+## v1.1.13 (May 20, 2021)
 
-#### Bug fix
+#### New feature
+- [91](https://github.com/Shopify/job-iteration/pull/91) - Add enumerator yielding batches as Active Record Relations
 
 ## v1.1.12 (April 19, 2021)
 

data/README.md

@@ -77,7 +77,28 @@ class BatchesJob < ApplicationJob
 
   def each_iteration(batch_of_comments, product_id)
     # batch_of_comments will contain batches of 100 records
-    Comment.where(id: batch_of_comments.map(&:id)).update_all(deleted: true)
+    batch_of_comments.each do |comment|
+      DeleteCommentJob.perform_later(comment)
+    end
+  end
+end
+```
+
+```ruby
+class BatchesAsRelationJob < ApplicationJob
+  include JobIteration::Iteration
+
+  def build_enumerator(product_id, cursor:)
+    enumerator_builder.active_record_on_batch_relations(
+      Product.find(product_id).comments,
+      cursor: cursor,
+      batch_size: 100,
+    )
+  end
+
+  def each_iteration(batch_of_comments, product_id)
+    # batch_of_comments will be a Comment::ActiveRecord_Relation
+    batch_of_comments.update_all(deleted: true)
   end
 end
 ```

data/lib/job-iteration/active_record_batch_enumerator.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module JobIteration
+  # Builds Batch Enumerator based on ActiveRecord Relation.
+  # @see EnumeratorBuilder
+  class ActiveRecordBatchEnumerator
+    include Enumerable
+
+    SQL_DATETIME_WITH_NSEC = "%Y-%m-%d %H:%M:%S.%N"
+
+    def initialize(relation, columns: nil, batch_size: 100, cursor: nil)
+      @batch_size = batch_size
+      @primary_key = "#{relation.table_name}.#{relation.primary_key}"
+      @columns = Array(columns&.map(&:to_s) || @primary_key)
+      @primary_key_index = @columns.index(@primary_key) || @columns.index(relation.primary_key)
+      @pluck_columns = if @primary_key_index
+        @columns
+      else
+        @columns.dup << @primary_key
+      end
+      @cursor = Array.wrap(cursor)
+      raise ArgumentError, "Must specify at least one column" if @columns.empty?
+      if relation.joins_values.present? && !@columns.all? { |column| column.to_s.include?(".") }
+        raise ArgumentError, "You need to specify fully-qualified columns if you join a table"
+      end
+
+      if relation.arel.orders.present? || relation.arel.taken.present?
+        raise ConditionNotSupportedError
+      end
+
+      @base_relation = relation.reorder(@columns.join(","))
+    end
+
+    def each
+      return to_enum { size } unless block_given?
+      while (relation = next_batch)
+        break if @cursor.nil?
+        yield relation, cursor_value
+      end
+    end
+
+    def size
+      @base_relation.count
+    end
+
+    private
+
+    def next_batch
+      relation = @base_relation.limit(@batch_size)
+      if conditions.any?
+        relation = relation.where(*conditions)
+      end
+
+      cursor_values, ids = relation.uncached do
+        pluck_columns(relation)
+      end
+
+      cursor = cursor_values.last
+      return unless cursor.present?
+      # The primary key was plucked, but original cursor did not include it, so we should remove it
+      cursor.pop unless @primary_key_index
+      @cursor = Array.wrap(cursor)
+
+      # Yields relations by selecting the primary keys of records in the batch.
+      # Post.where(published: nil) results in an enumerator of relations like: Post.where(ids: batch_of_ids)
+      @base_relation.where(@primary_key => ids)
+    end
+
+    def pluck_columns(relation)
+      if @pluck_columns.size == 1 # only the primary key
+        column_values = relation.pluck(*@pluck_columns)
+        return [column_values, column_values]
+      end
+
+      column_values = relation.pluck(*@pluck_columns)
+      primary_key_index = @primary_key_index || -1
+      primary_key_values = column_values.map { |values| values[primary_key_index] }
+
+      serialize_column_values!(column_values)
+      [column_values, primary_key_values]
+    end
+
+    def cursor_value
+      return @cursor.first if @cursor.size == 1
+      @cursor
+    end
+
+    def conditions
+      column_index = @cursor.size - 1
+      column = @columns[column_index]
+      where_clause = if @columns.size == @cursor.size
+        "#{column} > ?"
+      else
+        "#{column} >= ?"
+      end
+      while column_index > 0
+        column_index -= 1
+        column = @columns[column_index]
+        where_clause = "#{column} > ? OR (#{column} = ? AND (#{where_clause}))"
+      end
+      ret = @cursor.reduce([where_clause]) { |params, value| params << value << value }
+      ret.pop
+      ret
+    end
+
+    def serialize_column_values!(column_values)
+      column_values.map! { |values| values.map! { |value| column_value(value) } }
+    end
+
+    def column_value(value)
+      value.is_a?(Time) ? value.strftime(SQL_DATETIME_WITH_NSEC) : value
+    end
+  end
+end

data/lib/job-iteration/enumerator_builder.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require_relative "./active_record_batch_enumerator"
 require_relative "./active_record_enumerator"
 require_relative "./csv_enumerator"
 require_relative "./throttle_enumerator"
@@ -86,6 +87,11 @@ module JobIteration
     #        WHERE (created_at > '$LAST_CREATED_AT_CURSOR'
     #          OR (created_at = '$LAST_CREATED_AT_CURSOR' AND (id > '$LAST_ID_CURSOR')))
     #        ORDER BY created_at, id LIMIT 100
+    #
+    # As a result of this query pattern, if the values in these columns change for the records in scope during
+    # iteration, they may be skipped or yielded multiple times depending on the nature of the update and the
+    # cursor's value. If the value gets updated to a greater value than the cursor's value, it will get yielded
+    # again. Similarly, if the value gets updated to a lesser value than the curor's value, it will get skipped.
     def build_active_record_enumerator_on_records(scope, cursor:, **args)
       enum = build_active_record_enumerator(
         scope,
@@ -95,7 +101,7 @@ module JobIteration
       wrap(self, enum)
     end
 
-    # Builds Enumerator from Active Record Relation and enumerates on batches.
+    # Builds Enumerator from Active Record Relation and enumerates on batches of records.
     # Each Enumerator tick moves the cursor +batch_size+ rows forward.
     #
     # +batch_size:+ sets how many records will be fetched in one batch. Defaults to 100.
@@ -110,6 +116,16 @@ module JobIteration
       wrap(self, enum)
     end
 
+    # Builds Enumerator from Active Record Relation and enumerates on batches, yielding Active Record Relations.
+    # See documentation for #build_active_record_enumerator_on_batches.
+    def build_active_record_enumerator_on_batch_relations(scope, cursor:, **args)
+      JobIteration::ActiveRecordBatchEnumerator.new(
+        scope,
+        cursor: cursor,
+        **args
+      ).each
+    end
+
     def build_throttle_enumerator(enum, throttle_on:, backoff:)
       JobIteration::ThrottleEnumerator.new(
         enum,
@@ -124,6 +140,7 @@ module JobIteration
     alias_method :array, :build_array_enumerator
     alias_method :active_record_on_records, :build_active_record_enumerator_on_records
     alias_method :active_record_on_batches, :build_active_record_enumerator_on_batches
+    alias_method :active_record_on_batch_relations, :build_active_record_enumerator_on_batch_relations
     alias_method :throttle, :build_throttle_enumerator
 
     private

data/lib/job-iteration/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JobIteration
-  VERSION = "1.1.12"
+  VERSION = "1.1.13"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: job-iteration
 version: !ruby/object:Gem::Version
-  version: 1.1.12
+  version: 1.1.13
 platform: ruby
 authors:
 - Shopify
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-04-20 00:00:00.000000000 Z
+date: 2021-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -66,6 +66,7 @@ files:
 - guides/throttling.md
 - job-iteration.gemspec
 - lib/job-iteration.rb
+- lib/job-iteration/active_record_batch_enumerator.rb
 - lib/job-iteration/active_record_cursor.rb
 - lib/job-iteration/active_record_enumerator.rb
 - lib/job-iteration/csv_enumerator.rb
@@ -98,7 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.17
 signing_key: 
 specification_version: 4
 summary: Makes your background jobs interruptible and resumable.