activerecord 3.2.22.5 → 5.2.8

data/lib/active_record/relation/batches/batch_enumerator.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module ActiveRecord
+  module Batches
+    class BatchEnumerator
+      include Enumerable
+
+      def initialize(of: 1000, start: nil, finish: nil, relation:) #:nodoc:
+        @of       = of
+        @relation = relation
+        @start = start
+        @finish = finish
+      end
+
+      # Looping through a collection of records from the database (using the
+      # +all+ method, for example) is very inefficient since it will try to
+      # instantiate all the objects at once.
+      #
+      # In that case, batch processing methods allow you to work with the
+      # records in batches, thereby greatly reducing memory consumption.
+      #
+      #   Person.in_batches.each_record do |person|
+      #     person.do_awesome_stuff
+      #   end
+      #
+      #   Person.where("age > 21").in_batches(of: 10).each_record do |person|
+      #     person.party_all_night!
+      #   end
+      #
+      # If you do not provide a block to #each_record, it will return an Enumerator
+      # for chaining with other methods:
+      #
+      #   Person.in_batches.each_record.with_index do |person, index|
+      #     person.award_trophy(index + 1)
+      #   end
+      def each_record
+        return to_enum(:each_record) unless block_given?
+
+        @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: true).each do |relation|
+          relation.records.each { |record| yield record }
+        end
+      end
+
+      # Delegates #delete_all, #update_all, #destroy_all methods to each batch.
+      #
+      #   People.in_batches.delete_all
+      #   People.where('age < 10').in_batches.destroy_all
+      #   People.in_batches.update_all('age = age + 1')
+      [:delete_all, :update_all, :destroy_all].each do |method|
+        define_method(method) do |*args, &block|
+          @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false).each do |relation|
+            relation.send(method, *args, &block)
+          end
+        end
+      end
+
+      # Yields an ActiveRecord::Relation object for each batch of records.
+      #
+      #   Person.in_batches.each do |relation|
+      #     relation.update_all(awesome: true)
+      #   end
+      def each
+        enum = @relation.to_enum(:in_batches, of: @of, start: @start, finish: @finish, load: false)
+        return enum.each { |relation| yield relation } if block_given?
+        enum
+      end
+    end
+  end
+end

data/lib/active_record/relation/batches.rb

@@ -1,90 +1,287 @@
-require 'active_support/core_ext/object/blank'
+# frozen_string_literal: true
+
+require "active_record/relation/batches/batch_enumerator"
 
 module ActiveRecord
   module Batches
-    # Yields each record that was found by the find +options+. The find is
-    # performed by find_in_batches with a batch size of 1000 (or as
-    # specified by the <tt>:batch_size</tt> option).
+    ORDER_IGNORE_MESSAGE = "Scoped order is ignored, it's forced to be batch order."
+
+    # Looping through a collection of records from the database
+    # (using the Scoping::Named::ClassMethods.all method, for example)
+    # is very inefficient since it will try to instantiate all the objects at once.
+    #
+    # In that case, batch processing methods allow you to work
+    # with the records in batches, thereby greatly reducing memory consumption.
     #
-    # Example:
+    # The #find_each method uses #find_in_batches with a batch size of 1000 (or as
+    # specified by the +:batch_size+ option).
+    #
+    #   Person.find_each do |person|
+    #     person.do_awesome_stuff
+    #   end
     #
     #   Person.where("age > 21").find_each do |person|
     #     person.party_all_night!
     #   end
     #
-    # Note: This method is only intended to use for batch processing of
-    # large amounts of records that wouldn't fit in memory all at once. If
-    # you just need to loop over less than 1000 records, it's probably
-    # better just to use the regular find methods.
-    def find_each(options = {})
-      find_in_batches(options) do |records|
-        records.each { |record| yield record }
-      end
-    end
-
-    # Yields each batch of records that was found by the find +options+ as
-    # an array. The size of each batch is set by the <tt>:batch_size</tt>
-    # option; the default is 1000.
+    # If you do not provide a block to #find_each, it will return an Enumerator
+    # for chaining with other methods:
+    #
+    #   Person.find_each.with_index do |person, index|
+    #     person.award_trophy(index + 1)
+    #   end
     #
-    # You can control the starting point for the batch processing by
-    # supplying the <tt>:start</tt> option. This is especially useful if you
-    # want multiple workers dealing with the same processing queue. You can
-    # make worker 1 handle all the records between id 0 and 10,000 and
-    # worker 2 handle from 10,000 and beyond (by setting the <tt>:start</tt>
-    # option on that worker).
+    # ==== Options
+    # * <tt>:batch_size</tt> - Specifies the size of the batch. Defaults to 1000.
+    # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
+    # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
+    # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
+    #   an order is present in the relation.
     #
-    # It's not possible to set the order. That is automatically set to
+    # Limits are honored, and if present there is no requirement for the batch
+    # size: it can be less than, equal to, or greater than the limit.
+    #
+    # The options +start+ and +finish+ are especially useful if you want
+    # multiple workers dealing with the same processing queue. You can make
+    # worker 1 handle all the records between id 1 and 9999 and worker 2
+    # handle from 10000 and beyond by setting the +:start+ and +:finish+
+    # option on each worker.
+    #
+    #   # In worker 1, let's process until 9999 records.
+    #   Person.find_each(finish: 9_999) do |person|
+    #     person.party_all_night!
+    #   end
+    #
+    #   # In worker 2, let's process from record 10_000 and onwards.
+    #   Person.find_each(start: 10_000) do |person|
+    #     person.party_all_night!
+    #   end
+    #
+    # NOTE: It's not possible to set the order. That is automatically set to
     # ascending on the primary key ("id ASC") to make the batch ordering
-    # work. This also mean that this method only works with integer-based
-    # primary keys. You can't set the limit either, that's used to control
-    # the batch sizes.
+    # work. This also means that this method only works when the primary key is
+    # orderable (e.g. an integer or string).
     #
-    # Example:
+    # NOTE: By its nature, batch processing is subject to race conditions if
+    # other processes are modifying the database.
+    def find_each(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil)
+      if block_given?
+        find_in_batches(start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do |records|
+          records.each { |record| yield record }
+        end
+      else
+        enum_for(:find_each, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do
+          relation = self
+          apply_limits(relation, start, finish).size
+        end
+      end
+    end
+
+    # Yields each batch of records that was found by the find options as
+    # an array.
     #
     #   Person.where("age > 21").find_in_batches do |group|
     #     sleep(50) # Make sure it doesn't get too crowded in there!
     #     group.each { |person| person.party_all_night! }
     #   end
-    def find_in_batches(options = {})
+    #
+    # If you do not provide a block to #find_in_batches, it will return an Enumerator
+    # for chaining with other methods:
+    #
+    #   Person.find_in_batches.with_index do |group, batch|
+    #     puts "Processing group ##{batch}"
+    #     group.each(&:recover_from_last_night!)
+    #   end
+    #
+    # To be yielded each record one by one, use #find_each instead.
+    #
+    # ==== Options
+    # * <tt>:batch_size</tt> - Specifies the size of the batch. Defaults to 1000.
+    # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
+    # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
+    # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
+    #   an order is present in the relation.
+    #
+    # Limits are honored, and if present there is no requirement for the batch
+    # size: it can be less than, equal to, or greater than the limit.
+    #
+    # The options +start+ and +finish+ are especially useful if you want
+    # multiple workers dealing with the same processing queue. You can make
+    # worker 1 handle all the records between id 1 and 9999 and worker 2
+    # handle from 10000 and beyond by setting the +:start+ and +:finish+
+    # option on each worker.
+    #
+    #   # Let's process from record 10_000 on.
+    #   Person.find_in_batches(start: 10_000) do |group|
+    #     group.each { |person| person.party_all_night! }
+    #   end
+    #
+    # NOTE: It's not possible to set the order. That is automatically set to
+    # ascending on the primary key ("id ASC") to make the batch ordering
+    # work. This also means that this method only works when the primary key is
+    # orderable (e.g. an integer or string).
+    #
+    # NOTE: By its nature, batch processing is subject to race conditions if
+    # other processes are modifying the database.
+    def find_in_batches(start: nil, finish: nil, batch_size: 1000, error_on_ignore: nil)
       relation = self
+      unless block_given?
+        return to_enum(:find_in_batches, start: start, finish: finish, batch_size: batch_size, error_on_ignore: error_on_ignore) do
+          total = apply_limits(relation, start, finish).size
+          (total - 1).div(batch_size) + 1
+        end
+      end
+
+      in_batches(of: batch_size, start: start, finish: finish, load: true, error_on_ignore: error_on_ignore) do |batch|
+        yield batch.to_a
+      end
+    end
 
-      unless arel.orders.blank? && arel.taken.blank?
-        ActiveRecord::Base.logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size")
+    # Yields ActiveRecord::Relation objects to work with a batch of records.
+    #
+    #   Person.where("age > 21").in_batches do |relation|
+    #     relation.delete_all
+    #     sleep(10) # Throttle the delete queries
+    #   end
+    #
+    # If you do not provide a block to #in_batches, it will return a
+    # BatchEnumerator which is enumerable.
+    #
+    #   Person.in_batches.each_with_index do |relation, batch_index|
+    #     puts "Processing relation ##{batch_index}"
+    #     relation.delete_all
+    #   end
+    #
+    # Examples of calling methods on the returned BatchEnumerator object:
+    #
+    #   Person.in_batches.delete_all
+    #   Person.in_batches.update_all(awesome: true)
+    #   Person.in_batches.each_record(&:party_all_night!)
+    #
+    # ==== Options
+    # * <tt>:of</tt> - Specifies the size of the batch. Defaults to 1000.
+    # * <tt>:load</tt> - Specifies if the relation should be loaded. Defaults to false.
+    # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value.
+    # * <tt>:finish</tt> - Specifies the primary key value to end at, inclusive of the value.
+    # * <tt>:error_on_ignore</tt> - Overrides the application config to specify if an error should be raised when
+    #   an order is present in the relation.
+    #
+    # Limits are honored, and if present there is no requirement for the batch
+    # size, it can be less than, equal, or greater than the limit.
+    #
+    # The options +start+ and +finish+ are especially useful if you want
+    # multiple workers dealing with the same processing queue. You can make
+    # worker 1 handle all the records between id 1 and 9999 and worker 2
+    # handle from 10000 and beyond by setting the +:start+ and +:finish+
+    # option on each worker.
+    #
+    #   # Let's process from record 10_000 on.
+    #   Person.in_batches(start: 10_000).update_all(awesome: true)
+    #
+    # An example of calling where query method on the relation:
+    #
+    #   Person.in_batches.each do |relation|
+    #     relation.update_all('age = age + 1')
+    #     relation.where('age > 21').update_all(should_party: true)
+    #     relation.where('age <= 21').delete_all
+    #   end
+    #
+    # NOTE: If you are going to iterate through each record, you should call
+    # #each_record on the yielded BatchEnumerator:
+    #
+    #   Person.in_batches.each_record(&:party_all_night!)
+    #
+    # NOTE: It's not possible to set the order. That is automatically set to
+    # ascending on the primary key ("id ASC") to make the batch ordering
+    # consistent. Therefore the primary key must be orderable, e.g. an integer
+    # or a string.
+    #
+    # NOTE: By its nature, batch processing is subject to race conditions if
+    # other processes are modifying the database.
+    def in_batches(of: 1000, start: nil, finish: nil, load: false, error_on_ignore: nil)
+      relation = self
+      unless block_given?
+        return BatchEnumerator.new(of: of, start: start, finish: finish, relation: self)
       end
 
-      if (finder_options = options.except(:start, :batch_size)).present?
-        raise "You can't specify an order, it's forced to be #{batch_order}" if options[:order].present?
-        raise "You can't specify a limit, it's forced to be the batch_size"  if options[:limit].present?
+      if arel.orders.present?
+        act_on_ignored_order(error_on_ignore)
+      end
 
-        relation = apply_finder_options(finder_options)
+      batch_limit = of
+      if limit_value
+        remaining   = limit_value
+        batch_limit = remaining if remaining < batch_limit
       end
 
-      start = options.delete(:start)
-      batch_size = options.delete(:batch_size) || 1000
+      relation = relation.reorder(batch_order).limit(batch_limit)
+      relation = apply_limits(relation, start, finish)
+      relation.skip_query_cache! # Retaining the results in the query cache would undermine the point of batching
+      batch_relation = relation
+
+      loop do
+        if load
+          records = batch_relation.records
+          ids = records.map(&:id)
+          yielded_relation = where(primary_key => ids)
+          yielded_relation.load_records(records)
+        else
+          ids = batch_relation.pluck(primary_key)
+          yielded_relation = where(primary_key => ids)
+        end
+
+        break if ids.empty?
 
-      relation = relation.reorder(batch_order).limit(batch_size)
-      records = start ? relation.where(table[primary_key].gteq(start)).to_a : relation.to_a
+        primary_key_offset = ids.last
+        raise ArgumentError.new("Primary key not included in the custom select clause") unless primary_key_offset
 
-      while records.any?
-        records_size = records.size
-        primary_key_offset = records.last.id
+        yield yielded_relation
 
-        yield records
+        break if ids.length < batch_limit
 
-        break if records_size < batch_size
+        if limit_value
+          remaining -= ids.length
 
-        if primary_key_offset
-          records = relation.where(table[primary_key].gt(primary_key_offset)).to_a
-        else
-          raise "Primary key not included in the custom select clause"
+          if remaining == 0
+            # Saves a useless iteration when the limit is a multiple of the
+            # batch size.
+            break
+          elsif remaining < batch_limit
+            relation = relation.limit(remaining)
+          end
         end
+
+        attr = Relation::QueryAttribute.new(primary_key, primary_key_offset, klass.type_for_attribute(primary_key))
+        batch_relation = relation.where(arel_attribute(primary_key).gt(Arel::Nodes::BindParam.new(attr)))
       end
     end
 
     private
 
-    def batch_order
-      "#{quoted_table_name}.#{quoted_primary_key} ASC"
-    end
+      def apply_limits(relation, start, finish)
+        if start
+          attr = Relation::QueryAttribute.new(primary_key, start, klass.type_for_attribute(primary_key))
+          relation = relation.where(arel_attribute(primary_key).gteq(Arel::Nodes::BindParam.new(attr)))
+        end
+        if finish
+          attr = Relation::QueryAttribute.new(primary_key, finish, klass.type_for_attribute(primary_key))
+          relation = relation.where(arel_attribute(primary_key).lteq(Arel::Nodes::BindParam.new(attr)))
+        end
+        relation
+      end
+
+      def batch_order
+        arel_attribute(primary_key).asc
+      end
+
+      def act_on_ignored_order(error_on_ignore)
+        raise_error = (error_on_ignore.nil? ? klass.error_on_ignored_order : error_on_ignore)
+
+        if raise_error
+          raise ArgumentError.new(ORDER_IGNORE_MESSAGE)
+        elsif logger
+          logger.warn(ORDER_IGNORE_MESSAGE)
+        end
+      end
   end
 end