activerecord 6.0.0 → 7.2.3

data/lib/active_record/relation/batches/batch_enumerator.rb

@@ -5,11 +5,27 @@ module ActiveRecord
     class BatchEnumerator
       include Enumerable
 
-      def initialize(of: 1000, start: nil, finish: nil, relation:) #:nodoc:
+      def initialize(of: 1000, start: nil, finish: nil, relation:, order: :asc, use_ranges: nil) # :nodoc:
         @of       = of
         @relation = relation
         @start = start
         @finish = finish
+        @order = order
+        @use_ranges = use_ranges
+      end
+
+      # The primary key value from which the BatchEnumerator starts, inclusive of the value.
+      attr_reader :start
+
+      # The primary key value at which the BatchEnumerator ends, inclusive of the value.
+      attr_reader :finish
+
+      # The relation from which the BatchEnumerator yields batches.
+      attr_reader :relation
+
+      # The size of the batches yielded by the BatchEnumerator.
+      def batch_size
+        @of
       end
 
       # Looping through a collection of records from the database (using the
@@ -33,24 +49,53 @@ module ActiveRecord
       #   Person.in_batches.each_record.with_index do |person, index|
       #     person.award_trophy(index + 1)
       #   end
-      def each_record
+      def each_record(&block)
         return to_enum(:each_record) unless block_given?
 
-        @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: true).each do |relation|
-          relation.records.each { |record| yield record }
+        @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: true, order: @order).each do |relation|
+          relation.records.each(&block)
+        end
+      end
+
+      # Deletes records in batches. Returns the total number of rows affected.
+      #
+      #   Person.in_batches.delete_all
+      #
+      # See Relation#delete_all for details of how each batch is deleted.
+      def delete_all
+        sum(&:delete_all)
+      end
+
+      # Updates records in batches. Returns the total number of rows affected.
+      #
+      #   Person.in_batches.update_all("age = age + 1")
+      #
+      # See Relation#update_all for details of how each batch is updated.
+      def update_all(updates)
+        sum do |relation|
+          relation.update_all(updates)
+        end
+      end
+
+      # Touches records in batches. Returns the total number of rows affected.
+      #
+      #   Person.in_batches.touch_all
+      #
+      # See Relation#touch_all for details of how each batch is touched.
+      def touch_all(...)
+        sum do |relation|
+          relation.touch_all(...)
         end
       end
 
-      # Delegates #delete_all, #update_all, #destroy_all methods to each batch.
+      # Destroys records in batches. Returns the total number of rows affected.
+      #
+      #   Person.where("age < 10").in_batches.destroy_all
       #
-      #   People.in_batches.delete_all
-      #   People.where('age < 10').in_batches.destroy_all
-      #   People.in_batches.update_all('age = age + 1')
-      [:delete_all, :update_all, :destroy_all].each do |method|
-        define_method(method) do |*args, &block|
-          @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false).each do |relation|
-            relation.send(method, *args, &block)
-          end
+      # See Relation#destroy_all for details of how each batch is destroyed.
+      def destroy_all
+        sum do |relation|
+          relation.destroy_all.count(&:destroyed?)
         end
       end
 
@@ -59,9 +104,9 @@ module ActiveRecord
       #   Person.in_batches.each do |relation|
       #     relation.update_all(awesome: true)
       #   end
-      def each
-        enum = @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false)
-        return enum.each { |relation| yield relation } if block_given?
+      def each(&block)
+        enum = @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false, order: @order, use_ranges: @use_ranges)
+        return enum.each(&block) if block_given?
         enum
       end
     end

data/lib/active_record/relation/batches.rb

@@ -3,8 +3,10 @@
 require "active_record/relation/batches/batch_enumerator"
 
 module ActiveRecord
+  # = Active Record \Batches
   module Batches
     ORDER_IGNORE_MESSAGE = "Scoped order is ignored, it's forced to be batch order."
+    DEFAULT_ORDER = :asc
 
     # Looping through a collection of records from the database
     # (using the Scoping::Named::ClassMethods.all method, for example)
@@ -37,6 +39,16 @@ module ActiveRecord
     # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
     # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
     #   an order is present in the relation.
+    # * <tt>:order</tt> - Specifies the primary key order (can be +:asc+ or +:desc+ or an array consisting
+    #   of :asc or :desc). Defaults to +:asc+.
+    #
+    #     class Order < ActiveRecord::Base
+    #       self.primary_key = [:id_1, :id_2]
+    #     end
+    #
+    #     Order.find_each(order: [:asc, :desc])
+    #
+    #   In the above code, +id_1+ is sorted in ascending order and +id_2+ in descending order.
     #
     # Limits are honored, and if present there is no requirement for the batch
     # size: it can be less than, equal to, or greater than the limit.
@@ -57,22 +69,22 @@ module ActiveRecord
     #     person.party_all_night!
     #   end
     #
-    # NOTE: It's not possible to set the order. That is automatically set to
-    # ascending on the primary key ("id ASC") to make the batch ordering
-    # work. This also means that this method only works when the primary key is
+    # NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
+    # ascending on the primary key ("id ASC").
+    # This also means that this method only works when the primary key is
     # orderable (e.g. an integer or string).
     #
     # NOTE: By its nature, batch processing is subject to race conditions if
     # other processes are modifying the database.
-    def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil)
+    def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, order: DEFAULT_ORDER, &block)
       if block_given?
-        find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do |records|
-          records.each { |record| yield record }
+        find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, order: order) do |records|
+          records.each(&block)
         end
       else
-        enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do
+        enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, order: order) do
           relation = self
-          apply_limits(relation, start, finish).size
+          apply_limits(relation, start, finish, build_batch_orders(order)).size
         end
       end
     end
@@ -101,6 +113,16 @@ module ActiveRecord
     # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
     # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
     #   an order is present in the relation.
+    # * <tt>:order</tt> - Specifies the primary key order (can be +:asc+ or +:desc+ or an array consisting
+    #   of :asc or :desc). Defaults to +:asc+.
+    #
+    #     class Order < ActiveRecord::Base
+    #       self.primary_key = [:id_1, :id_2]
+    #     end
+    #
+    #     Order.find_in_batches(order: [:asc, :desc])
+    #
+    #   In the above code, +id_1+ is sorted in ascending order and +id_2+ in descending order.
     #
     # Limits are honored, and if present there is no requirement for the batch
     # size: it can be less than, equal to, or greater than the limit.
@@ -116,23 +138,23 @@ module ActiveRecord
     #     group.each { |person| person.party_all_night! }
     #   end
     #
-    # NOTE: It's not possible to set the order. That is automatically set to
-    # ascending on the primary key ("id ASC") to make the batch ordering
-    # work. This also means that this method only works when the primary key is
+    # NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
+    # ascending on the primary key ("id ASC").
+    # This also means that this method only works when the primary key is
     # orderable (e.g. an integer or string).
     #
     # NOTE: By its nature, batch processing is subject to race conditions if
     # other processes are modifying the database.
-    def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil)
+    def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil, order: DEFAULT_ORDER)
       relation = self
       unless block_given?
-        return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do
-          total = apply_limits(relation, start, finish).size
+        return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore, order: order) do
+          total = apply_limits(relation, start, finish, build_batch_orders(order)).size
           (total - 1).div(batch_size) + 1
         end
       end
 
-      in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore) do |batch|
+      in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore, order: order) do |batch|
         yield batch.to_a
       end
     end
@@ -165,6 +187,22 @@ module ActiveRecord
     # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
     # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
     #   an order is present in the relation.
+    # * <tt>:order</tt> - Specifies the primary key order (can be +:asc+ or +:desc+ or an array consisting
+    #   of :asc or :desc). Defaults to +:asc+.
+    #
+    #     class Order < ActiveRecord::Base
+    #       self.primary_key = [:id_1, :id_2]
+    #     end
+    #
+    #     Order.in_batches(order: [:asc, :desc])
+    #
+    #   In the above code, +id_1+ is sorted in ascending order and +id_2+ in descending order.
+    #
+    # * <tt>:use_ranges</tt> - Specifies whether to use range iteration (id >= x AND id <= y).
+    #   It can make iterating over the whole or almost whole tables several times faster.
+    #   Only whole table iterations use this style of iteration by default. You can disable this behavior by passing +false+.
+    #   If you iterate over the table and the only condition is, e.g., <tt>archived_at: nil</tt> (and only a tiny fraction
+    #   of the records are archived), it makes sense to opt in to this approach.
     #
     # Limits are honored, and if present there is no requirement for the batch
     # size, it can be less than, equal, or greater than the limit.
@@ -191,94 +229,101 @@ module ActiveRecord
     #
     #   Person.in_batches.each_record(&:party_all_night!)
     #
-    # NOTE: It's not possible to set the order. That is automatically set to
-    # ascending on the primary key ("id ASC") to make the batch ordering
-    # consistent. Therefore the primary key must be orderable, e.g. an integer
-    # or a string.
+    # NOTE: Order can be ascending (:asc) or descending (:desc). It is automatically set to
+    # ascending on the primary key ("id ASC").
+    # This also means that this method only works when the primary key is
+    # orderable (e.g. an integer or string).
     #
     # NOTE: By its nature, batch processing is subject to race conditions if
     # other processes are modifying the database.
-    def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil)
-      relation = self
-      unless block_given?
-        return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self)
+    def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil, order: DEFAULT_ORDER, use_ranges: nil, &block)
+      unless Array(order).all? { |ord| [:asc, :desc].include?(ord) }
+        raise ArgumentError, ":order must be :asc or :desc or an array consisting of :asc or :desc, got #{order.inspect}"
       end
 
       if arel.orders.present?
         act_on_ignored_order(error_on_ignore)
       end
 
+      unless block
+        return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self, order: order, use_ranges: use_ranges)
+      end
+
       batch_limit = of
+
       if limit_value
         remaining   = limit_value
         batch_limit = remaining if remaining < batch_limit
       end
 
-      relation = relation.reorder(batch_order).limit(batch_limit)
-      relation = apply_limits(relation, start, finish)
-      relation.skip_query_cache! # Retaining the results in the query cache would undermine the point of batching
-      batch_relation = relation
-
-      loop do
-        if load
-          records = batch_relation.records
-          ids = records.map(&:id)
-          yielded_relation = where(primary_key => ids)
-          yielded_relation.load_records(records)
-        else
-          ids = batch_relation.pluck(primary_key)
-          yielded_relation = where(primary_key => ids)
-        end
-
-        break if ids.empty?
-
-        primary_key_offset = ids.last
-        raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset
-
-        yield yielded_relation
-
-        break if ids.length < batch_limit
-
-        if limit_value
-          remaining -= ids.length
-
-          if remaining == 0
-            # Saves a useless iteration when the limit is a multiple of the
-            # batch size.
-            break
-          elsif remaining < batch_limit
-            relation = relation.limit(remaining)
-          end
-        end
-
-        batch_relation = relation.where(
-          bind_attribute(primary_key, primary_key_offset) { |attr, bind| attr.gt(bind) }
+      if self.loaded?
+        batch_on_loaded_relation(
+          relation: self,
+          start: start,
+          finish: finish,
+          order: order,
+          batch_limit: batch_limit,
+          &block
+        )
+      else
+        batch_on_unloaded_relation(
+          relation: self,
+          start: start,
+          finish: finish,
+          load: load,
+          order: order,
+          use_ranges: use_ranges,
+          remaining: remaining,
+          batch_limit: batch_limit,
+          &block
         )
       end
     end
 
     private
-
-      def apply_limits(relation, start, finish)
-        relation = apply_start_limit(relation, start) if start
-        relation = apply_finish_limit(relation, finish) if finish
+      def apply_limits(relation, start, finish, batch_orders)
+        relation = apply_start_limit(relation, start, batch_orders) if start
+        relation = apply_finish_limit(relation, finish, batch_orders) if finish
         relation
       end
 
-      def apply_start_limit(relation, start)
-        relation.where(bind_attribute(primary_key, start) { |attr, bind| attr.gteq(bind) })
+      def apply_start_limit(relation, start, batch_orders)
+        operators = batch_orders.map do |_column, order|
+          order == :desc ? :lteq : :gteq
+        end
+        batch_condition(relation, primary_key, start, operators)
       end
 
-      def apply_finish_limit(relation, finish)
-        relation.where(bind_attribute(primary_key, finish) { |attr, bind| attr.lteq(bind) })
+      def apply_finish_limit(relation, finish, batch_orders)
+        operators = batch_orders.map do |_column, order|
+          order == :desc ? :gteq : :lteq
+        end
+        batch_condition(relation, primary_key, finish, operators)
       end
 
-      def batch_order
-        arel_attribute(primary_key).asc
+      def batch_condition(relation, columns, values, operators)
+        cursor_positions = Array(columns).zip(Array(values), operators)
+
+        first_clause_column, first_clause_value, operator = cursor_positions.pop
+        where_clause = predicate_builder[first_clause_column, first_clause_value, operator]
+
+        cursor_positions.reverse_each do |column_name, value, operator|
+          where_clause = predicate_builder[column_name, value, operator == :lteq ? :lt : :gt].or(
+            predicate_builder[column_name, value, :eq].and(where_clause)
+          )
+        end
+
+        relation.where(where_clause)
+      end
+
+      def build_batch_orders(order)
+        get_the_order_of_primary_key(order).map do |column, ord|
+          [column, ord || DEFAULT_ORDER]
+        end
       end
 
       def act_on_ignored_order(error_on_ignore)
-        raise_error = (error_on_ignore.nil? ? klass.error_on_ignored_order : error_on_ignore)
+        raise_error = (error_on_ignore.nil? ? ActiveRecord.error_on_ignored_order : error_on_ignore)
 
         if raise_error
           raise ArgumentError.new(ORDER_IGNORE_MESSAGE)
@@ -286,5 +331,101 @@ module ActiveRecord
           logger.warn(ORDER_IGNORE_MESSAGE)
         end
       end
+
+      def get_the_order_of_primary_key(order)
+        Array(primary_key).zip(Array(order))
+      end
+
+      def batch_on_loaded_relation(relation:, start:, finish:, order:, batch_limit:)
+        records = relation.to_a
+
+        if start || finish
+          records = records.filter do |record|
+            id = record.id
+
+            if order == :asc
+              (start.nil? || id >= start) && (finish.nil? || id <= finish)
+            else
+              (start.nil? || id <= start) && (finish.nil? || id >= finish)
+            end
+          end
+        end
+
+        records.sort_by!(&:id)
+
+        if order == :desc
+          records.reverse!
+        end
+
+        records.each_slice(batch_limit) do |subrecords|
+          subrelation = relation.spawn
+          subrelation.load_records(subrecords)
+
+          yield subrelation
+        end
+
+        nil
+      end
+
+      def batch_on_unloaded_relation(relation:, start:, finish:, load:, order:, use_ranges:, remaining:, batch_limit:)
+        batch_orders = build_batch_orders(order)
+        relation = relation.reorder(batch_orders.to_h).limit(batch_limit)
+        relation = apply_limits(relation, start, finish, batch_orders)
+        relation.skip_query_cache! # Retaining the results in the query cache would undermine the point of batching
+        batch_relation = relation
+        empty_scope = to_sql == klass.unscoped.all.to_sql
+
+        loop do
+          if load
+            records = batch_relation.records
+            ids = records.map(&:id)
+            yielded_relation = where(primary_key => ids)
+            yielded_relation.load_records(records)
+          elsif (empty_scope && use_ranges != false) || use_ranges
+            ids = batch_relation.ids
+            finish = ids.last
+            if finish
+              yielded_relation = apply_finish_limit(batch_relation, finish, batch_orders)
+              yielded_relation = yielded_relation.except(:limit, :order)
+              yielded_relation.skip_query_cache!(false)
+            end
+          else
+            ids = batch_relation.ids
+            yielded_relation = where(primary_key => ids)
+          end
+
+          break if ids.empty?
+
+          primary_key_offset = ids.last
+          raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset
+
+          yield yielded_relation
+
+          break if ids.length < batch_limit
+
+          if limit_value
+            remaining -= ids.length
+
+            if remaining == 0
+              # Saves a useless iteration when the limit is a multiple of the
+              # batch size.
+              break
+            elsif remaining < batch_limit
+              relation = relation.limit(remaining)
+            end
+          end
+
+          batch_orders_copy = batch_orders.dup
+          _last_column, last_order = batch_orders_copy.pop
+          operators = batch_orders_copy.map do |_column, order|
+            order == :desc ? :lteq : :gteq
+          end
+          operators << (last_order == :desc ? :lt : :gt)
+
+          batch_relation = batch_condition(relation, primary_key, primary_key_offset, operators)
+        end
+
+        nil
+      end
   end
 end