occams-record 1.3.0 → 1.4.0.pre.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 95a5656a0aa870ae2458e549442d68f8f685388d2f561321daae428509bdac6e
-  data.tar.gz: 9b5c0edddab969fd9ba3140e028dd4d36d4408fe079c5e8987be8b35a3263d4e
+  metadata.gz: 662bd65e909a1d957ebdbbd2d3bec3fe96fb45aac2117a786c9a3ad02b03259b
+  data.tar.gz: abeb69ff706dbb11f3fae3672ae60c407080f418fce7330779a7b550915aa886
 SHA512:
-  metadata.gz: 1fb09da919637085f3673dffa20416dbbcfcd5f03d0b9d4efbcc074f26a076b44a286a10da6f311822dda843b4480f631dc93d8f6e7cfe9818998e933bd0c099
-  data.tar.gz: 9f3dc78c47a39f396ee393e8c9a472dedd3e436db00cafd6e7773f8137e143fb5c5b86bffa7db204b63d170567285c5baf04928b4a4405aead081115c5b77bf1
+  metadata.gz: 8981d9c820b64ebeca28dcbe3ab7ac44e35d8ec7e14df201e890ccf6cf1cf1888d6f78ba6cac001f1a9c458f604189b1aafaf7ff644288499faf5d5afeb896c3
+  data.tar.gz: cdd61c50313a964db4ebdb271dbb875fc4ea888dd0b57df78be78d043e655e04e821e0cef8d9b60034e90866289a0d764f72a7728a18f46c778e511101b72f4a

data/README.md

@@ -300,6 +300,9 @@ bundle exec rake test
 
 # test against Postgres
 TEST_DATABASE_URL=postgres://postgres@localhost:5432/occams_record bundle exec rake test
+
+# test against MySQL
+TEST_DATABASE_URL=mysql2://root:@127.0.0.1:3306/occams_record bundle exec rake test
 ```
 
 **Test against all supported ActiveRecord versions**

data/lib/occams-record/batches/cursor_helpers.rb

@@ -0,0 +1,59 @@
+module OccamsRecord
+  module Batches
+    module CursorHelpers
+      #
+      # Loads records in batches of N and yields each record to a block (if given). If no block is given,
+      # returns an Enumerator.
+      #
+      # NOTE Unlike find_each, batches are loaded using a cursor, which offers better performance.
+      # Postgres only. See the docs for OccamsRecord::Cursor for more details.
+      #
+      # @param batch_size [Integer] fetch this many rows at once
+      # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+      # @yield [OccamsRecord::Results::Row]
+      # @return [Enumerator] will yield each record
+      #
+      def find_each_with_cursor(batch_size: 1000, use_transaction: true)
+        enum = Enumerator.new { |y|
+          cursor.open(use_transaction: use_transaction) { |c|
+            c.each(batch_size: batch_size) { |record|
+              y.yield record
+            }
+          }
+        }
+        if block_given?
+          enum.each { |record| yield record }
+        else
+          enum
+        end
+      end
+
+      #
+      # Loads records in batches of N and yields each batch to a block (if given). If no block is given,
+      # returns an Enumerator.
+      #
+      # NOTE Unlike find_in_batches, batches are loaded using a cursor, which offers better performance.
+      # Postgres only. See the docs for OccamsRecord::Cursor for more details.
+      #
+      # @param batch_size [Integer] fetch this many rows at once
+      # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+      # @yield [OccamsRecord::Results::Row]
+      # @return [Enumerator] will yield each record
+      #
+      def find_in_batches_with_cursor(batch_size: 1000, use_transaction: true)
+        enum = Enumerator.new { |y|
+          cursor.open(use_transaction: use_transaction) { |c|
+            c.each_batch(batch_size: batch_size) { |batch|
+              y.yield batch
+            }
+          }
+        } 
+        if block_given?
+          enum.each { |batch| yield batch }
+        else
+          enum
+        end
+      end
+    end
+  end
+end

data/lib/occams-record/batches/offset_limit/raw_query.rb

@@ -0,0 +1,53 @@
+module OccamsRecord
+  module Batches
+    module OffsetLimit
+      class RawQuery
+        def initialize(conn, sql, binds, use: nil, query_logger: nil, eager_loaders: nil)
+          @conn, @sql, @binds = conn, sql, binds
+          @use, @query_logger, @eager_loaders = use, query_logger, eager_loaders
+
+          unless @sql =~ /LIMIT\s+%\{batch_limit\}/i and @sql =~ /OFFSET\s+%\{batch_offset\}/i
+            raise ArgumentError, "When using find_each/find_in_batches you must specify 'LIMIT %{batch_limit} OFFSET %{batch_offset}'. SQL statement: #{@sql}"
+          end
+        end
+
+        #
+        # Returns an Enumerator that yields batches of records, of size "of".
+        # The SQL string must include 'LIMIT %{batch_limit} OFFSET %{batch_offset}'.
+        # The bind values will be provided by OccamsRecord.
+        #
+        # @param batch_size [Integer] batch size
+        # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+        # @return [Enumerator] yields batches
+        #
+        def enum(batch_size:, use_transaction: true)
+          Enumerator.new do |y|
+            if use_transaction and @conn.open_transactions == 0
+              @conn.transaction {
+                run_batches y, batch_size
+              }
+            else
+              run_batches y, batch_size
+            end
+          end
+        end
+
+        private
+
+        def run_batches(y, of)
+          offset = 0
+          loop do
+            results = ::OccamsRecord::RawQuery.new(@sql, @binds.merge({
+              batch_limit: of,
+              batch_offset: offset,
+            }), use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders).run
+
+            y.yield results if results.any?
+            break if results.size < of
+            offset += results.size
+          end
+        end
+      end
+    end
+  end
+end

data/lib/occams-record/batches/offset_limit/scoped.rb

@@ -0,0 +1,72 @@
+module OccamsRecord
+  #
+  # Methods for building batch finding methods. It expects "model" and "scope" methods to be present.
+  #
+  module Batches
+    module OffsetLimit
+      class Scoped
+        def initialize(model, scope, use: nil, query_logger: nil, eager_loaders: nil)
+          @model, @scope = model, scope
+          @use, @query_logger, @eager_loaders = use, query_logger, eager_loaders
+        end
+
+        #
+        # Returns an Enumerator that yields batches of records, of size "of".
+        # NOTE ActiveRecord 5+ provides the 'in_batches' method to do something
+        # similiar, although 4.2 does not. Also it does not respect ORDER BY,
+        # whereas this does.
+        #
+        # @param batch_size [Integer] batch size
+        # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+        # @param append_order_by [String] Append this column to ORDER BY to ensure consistent results. Defaults to the primary key. Pass false to disable.
+        # @return [Enumerator] yields batches
+        #
+        def enum(batch_size:, use_transaction: true, append_order_by: nil)
+          append_order =
+            case append_order_by
+            when false then nil
+            when nil then @model.primary_key
+            else append_order_by
+            end
+
+          Enumerator.new do |y|
+            if use_transaction and @model.connection.open_transactions == 0
+              @model.connection.transaction {
+                run_batches y, batch_size, append_order
+              }
+            else
+              run_batches y, batch_size, append_order
+            end
+          end
+        end
+
+        private
+
+        def run_batches(y, of, append_order_by = nil)
+          limit = @scope.limit_value
+          batch_size = limit && limit < of ? limit : of
+
+          offset = @scope.offset_value || 0
+          out_of_records, count = false, 0
+          order_by =
+            if append_order_by
+              append_order_by.to_s == @model.primary_key.to_s ? append_order_by.to_sym : append_order_by
+            end
+
+          until out_of_records
+            l = limit && batch_size > limit - count ? limit - count : batch_size
+            q = @scope
+            q = q.order(order_by) if order_by
+            q = q.offset(offset).limit(l)
+            results = ::OccamsRecord::Query.new(q, use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders).run
+
+            y.yield results if results.any?
+            count += results.size
+            offset += results.size
+            out_of_records = results.size < batch_size || (limit && count >= limit)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/occams-record/cursor.rb

@@ -0,0 +1,237 @@
+require 'securerandom'
+
+module OccamsRecord
+#
+# An interface to database cursors. Supported databases:
+#   * PostgreSQL
+#
+  class Cursor
+    # @private
+    SCROLL = {
+      true => "SCROLL",
+      false => "NO SCROLL",
+      nil => "",
+    }.freeze
+
+    # @private
+    HOLD = {
+      true => "WITH HOLD",
+      false => "WITHOUT HOLD",
+      nil => "",
+    }.freeze
+
+    # @private
+    DIRECTIONS = {
+      next: "NEXT",
+      prior: "PRIOR",
+      first: "FIRST",
+      last: "LAST",
+      absolute: "ABSOLUTE",
+      relative: "RELATIVE",
+      forward: "FORWARD",
+      backward: "BACKWARD",
+    }.freeze
+
+    # @return [ActiveRecord::Connection]
+    attr_reader :conn
+
+    # Name of the cursor
+    # @return [String]
+    attr_reader :name
+
+    # Name of the cursor (safely SQL-escaped)
+    # @return [String]
+    attr_reader :quoted_name
+
+    #
+    # Initializes a new Cursor. NOTE all operations must be performed within a block passed to #open.
+    #
+    # While you CAN manually initialize a cursor, it's more common to get one via OccamsRecord::Query#cursor
+    # or OccamsRecord::RawQuery#cursor.
+    #
+    # @param conn [ActiveRecord::Connection]
+    # @param sql [String] The query to run
+    # @param name [String] Specify a name for the cursor (defaults to a random name)
+    # @param scroll [Boolean] true = SCROLL, false = NO SCROLL, nil = default behavior of DB
+    # @param hold [Boolean] true = WITH HOLD, false = WITHOUT HOLD, nil = default behavior of DB
+    # @param use [Array<Module>] optional Module to include in the result class (single or array)
+    # @param query_logger [Array] (optional) an array into which all queries will be inserted for logging/debug purposes
+    # @param eager_loaders [OccamsRecord::EagerLoaders::Context]
+    #
+    def initialize(conn, sql, name: nil, scroll: nil, hold: nil, use: nil, query_logger: nil, eager_loaders: nil)
+      @conn, @sql = conn, sql
+      @scroll = SCROLL.fetch(scroll)
+      @hold = HOLD.fetch(hold)
+      @use, @query_logger, @eager_loaders = use, query_logger, eager_loaders
+      @name = name || "occams_cursor_#{SecureRandom.hex 4}"
+      @quoted_name = conn.quote_table_name(@name)
+    end
+
+    #
+    # Declares and opens the cursor, runs the given block (yielding self), and closes the cursor.
+    #
+    #   cursor.open do |c|
+    #     c.fetch :forward, 100
+    #   end
+    #
+    # @param use_transaction [Boolean] When true, ensures it's wrapped in a transaction
+    # @yield [self]
+    # @return the value returned by the block
+    #
+    def open(use_transaction: true)
+      raise ArgumentError, "A block is required" unless block_given?
+      if use_transaction and conn.open_transactions == 0
+        conn.transaction {
+          perform { yield self }
+        }
+      else
+        perform { yield self }
+      end
+    end
+
+    #
+    # Loads records in batches of N and yields each record to a block (if given). If no block is given,
+    # returns an Enumerator.
+    #
+    #   cursor.open do |c|
+    #     c.each do |record|
+    #       ...
+    #     end
+    #   end
+    #
+    # @param batch_size [Integer] fetch this many rows at once
+    #
+    def each(batch_size: 1000)
+      enum = Enumerator.new { |y|
+        each_batch(batch_size: batch_size).each { |batch|
+          batch.each { |record| y.yield record }
+        }
+      }
+      if block_given?
+        enum.each { |record| yield record }
+      else
+        enum
+      end
+    end
+
+    #
+    # Loads records in batches of N and yields each batch to a block (if given). If no block is given,
+    # returns an Enumerator.
+    #
+    #   cursor.open do |c|
+    #     c.each_batch do |batch|
+    #       ...
+    #     end
+    #   end
+    #
+    # @param batch_size [Integer] fetch this many rows at once
+    #
+    def each_batch(batch_size: 1000)
+      enum = Enumerator.new { |y|
+        out_of_records = false
+        until out_of_records
+          results = fetch :forward, batch_size
+          y.yield results if results.any?
+          out_of_records = results.size < batch_size
+        end
+      }
+      if block_given?
+        enum.each { |batch| yield batch }
+      else
+        enum
+      end
+    end
+
+    #
+    # Fetch records in the given direction.
+    #
+    #   cursor.open do |c|
+    #     c.fetch :forward, 100
+    #     ...
+    #   end
+    #
+    # @param direction [Symbol] :next, :prior, :first, :last, :absolute, :relative, :forward or :backward
+    # @param num [Integer] number of rows to fetch (optional for some directions)
+    # @return [OccamsRecord::Results::Row]
+    #
+    def fetch(direction, num = nil)
+      query "FETCH %{dir} %{num} FROM %{name}".freeze % {
+        dir: DIRECTIONS.fetch(direction),
+        num: num&.to_i,
+        name: @quoted_name,
+      }
+    end
+
+    #
+    # Move the cursor the given direction.
+    #
+    #   cursor.open do |c|
+    #     ...
+    #     c.move :backward, 100
+    #     ...
+    #   end
+    #
+    # @param direction [Symbol] :next, :prior, :first, :last, :absolute, :relative, :forward or :backward
+    # @param num [Integer] number of rows to move (optional for some directions)
+    #
+    def move(direction, num = nil)
+      query "MOVE %{dir} %{num} FROM %{name}".freeze % {
+        dir: DIRECTIONS.fetch(direction),
+        num: num&.to_i,
+        name: @quoted_name,
+      }
+    end
+
+    #
+    # Run an arbitrary query on the cursor. Use 'binds' to escape inputs.
+    #
+    #   cursor.open do |c|
+    #     c.query("FETCH FORWARD %{num} FOR #{c.quoted_name}", {num: 100})
+    #     ...
+    #   end
+    #
+    def query(sql, binds = {})
+      ::OccamsRecord::RawQuery.new(sql, binds, use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders, connection: conn).run
+    end
+
+    #
+    # Run an arbitrary command on the cursor. Use 'binds' to escape inputs.
+    #
+    #   cursor.open do |c|
+    #     c.execute("MOVE FORWARD %{num} FOR #{c.quoted_name}", {num: 100})
+    #     ...
+    #   end
+    #
+    def execute(sql, binds = {})
+      conn.execute(sql % binds.reduce({}) { |acc, (key, val)|
+        acc[key] = conn.quote(val)
+        acc
+      })
+    end
+
+    private
+
+    def perform
+      ex = nil
+      conn.execute "DECLARE %{name} %{scroll} CURSOR %{hold} FOR %{query}".freeze % {
+        name: @quoted_name,
+        scroll: @scroll,
+        hold: @hold,
+        query: @sql,
+      }
+      yield
+    rescue => e
+      ex = e
+      raise ex
+    ensure
+      begin
+        conn.execute "CLOSE %{name}".freeze % {name: @quoted_name}
+      rescue => e
+        # Don't let an error from CLOSE (like a dead transaction) hide what lead to the error with CLOSE (like bad SQL that raised an error and aborted the transaction)
+        raise ex || e
+      else
+        raise ex if ex
+      end
+    end
+  end
+end

data/lib/occams-record/query.rb

@@ -1,5 +1,3 @@
-require 'occams-record/batches'
-
 module OccamsRecord
   #
   # Starts building a OccamsRecord::Query. Pass it a scope from any of ActiveRecord's query builder
@@ -36,7 +34,7 @@ module OccamsRecord
     # @return [ActiveRecord::Relation] scope for building the main SQL query
     attr_reader :scope
 
-    include Batches
+    include OccamsRecord::Batches::CursorHelpers
     include EagerLoaders::Builder
     include Enumerable
     include Measureable
@@ -154,5 +152,75 @@ module OccamsRecord
         to_a.each
       end
     end
+
+    #
+    # Load records in batches of N and yield each record to a block if given. If no block is given,
+    # returns an Enumerator.
+    #
+    # NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
+    # to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
+    # of a transaction.
+    #
+    # @param batch_size [Integer]
+    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+    # @param append_order_by [String] Append this column to ORDER BY to ensure consistent results. Defaults to the primary key. Pass false to disable.
+    # @yield [OccamsRecord::Results::Row]
+    # @return [Enumerator] will yield each record
+    #
+    def find_each(batch_size: 1000, use_transaction: true, append_order_by: nil)
+      enum = Enumerator.new { |y|
+        find_in_batches(batch_size: 1000, use_transaction: use_transaction, append_order_by: append_order_by).each { |batch|
+          batch.each { |record| y.yield record }
+        }
+      }
+      if block_given?
+        enum.each { |record| yield record }
+      else
+        enum
+      end
+    end
+
+    #
+    # Load records in batches of N and yield each batch to a block if given.
+    # If no block is given, returns an Enumerator.
+    #
+    # NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
+    # to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
+    # of a transaction.
+    #
+    # @param batch_size [Integer]
+    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+    # @param append_order_by [String] Append this column to ORDER BY to ensure consistent results. Defaults to the primary key. Pass false to disable.
+    # @yield [OccamsRecord::Results::Row]
+    # @return [Enumerator] will yield each batch
+    #
+    def find_in_batches(batch_size: 1000, use_transaction: true, append_order_by: nil)
+      enum = Batches::OffsetLimit::Scoped
+        .new(model, scope, use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders)
+        .enum(batch_size: batch_size, use_transaction: use_transaction, append_order_by: append_order_by)
+      if block_given?
+        enum.each { |batch| yield batch }
+      else
+        enum
+      end
+    end
+
+    #
+    # Returns a cursor you can open and perform operations on. A lower-level alternative to 
+    # find_each_with_cursor and find_in_batches_with_cursor.
+    #
+    # NOTE Postgres only. See the docs for OccamsRecord::Cursor for more details.
+    #
+    # @param name [String] Specify a name for the cursor (defaults to a random name)
+    # @param scroll [Boolean] true = SCROLL, false = NO SCROLL, nil = default behavior of DB
+    # @param hold [Boolean] true = WITH HOLD, false = WITHOUT HOLD, nil = default behavior of DB
+    # @return [OccamsRecord::Cursor]
+    #
+    def cursor(name: nil, scroll: nil, hold: nil)
+      Cursor.new(model.connection, scope.to_sql,
+        name: name, scroll: scroll, hold: hold,
+        use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders,
+      )
+    end
   end
 end

data/lib/occams-record/raw_query.rb

@@ -61,7 +61,7 @@ module OccamsRecord
     # @return [Hash]
     attr_reader :binds
 
-    include Batches
+    include OccamsRecord::Batches::CursorHelpers
     include EagerLoaders::Builder
     include Enumerable
     include Measureable
@@ -75,13 +75,15 @@ module OccamsRecord
     # @param eager_loaders [OccamsRecord::EagerLoaders::Context]
     # @param query_logger [Array] (optional) an array into which all queries will be inserted for logging/debug purposes
     # @param measurements [Array]
+    # @param connection
     #
-    def initialize(sql, binds, use: nil, eager_loaders: nil, query_logger: nil, measurements: nil)
+    def initialize(sql, binds, use: nil, eager_loaders: nil, query_logger: nil, measurements: nil, connection: nil)
       @sql = sql
       @binds = binds
       @use = use
       @eager_loaders = eager_loaders || EagerLoaders::Context.new
       @query_logger, @measurements = query_logger, measurements
+      @conn = connection
     end
 
     #
@@ -139,6 +141,74 @@ module OccamsRecord
       end
     end
 
+    #
+    # Load records in batches of N and yield each record to a block if given. If no block is given,
+    # returns an Enumerator.
+    #
+    # NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
+    # to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
+    # of a transaction.
+    #
+    # @param batch_size [Integer]
+    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+    # @yield [OccamsRecord::Results::Row]
+    # @return [Enumerator] will yield each record
+    #
+    def find_each(batch_size: 1000, use_transaction: true)
+      enum = Enumerator.new { |y|
+        find_in_batches(batch_size: batch_size, use_transaction: use_transaction).each { |batch|
+          batch.each { |record| y.yield record }
+        }
+      }
+      if block_given?
+        enum.each { |record| yield record }
+      else
+        enum
+      end
+    end
+
+    #
+    # Load records in batches of N and yield each batch to a block if given.
+    # If no block is given, returns an Enumerator.
+    #
+    # NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
+    # to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
+    # of a transaction.
+    #
+    # @param batch_size [Integer]
+    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+    # @yield [OccamsRecord::Results::Row]
+    # @return [Enumerator] will yield each batch
+    #
+    def find_in_batches(batch_size: 1000, use_transaction: true)
+      enum = Batches::OffsetLimit::RawQuery
+        .new(conn, @sql, @binds, use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders)
+        .enum(batch_size: batch_size, use_transaction: use_transaction)
+      if block_given?
+        enum.each { |batch| yield batch }
+      else
+        enum
+      end
+    end
+
+    #
+    # Returns a cursor you can open and perform operations on. A lower-level alternative to 
+    # find_each_with_cursor and find_in_batches_with_cursor.
+    #
+    # NOTE Postgres only. See the docs for OccamsRecord::Cursor for more details.
+    #
+    # @param name [String] Specify a name for the cursor (defaults to a random name)
+    # @param scroll [Boolean] true = SCROLL, false = NO SCROLL, nil = default behavior of DB
+    # @param hold [Boolean] true = WITH HOLD, false = WITHOUT HOLD, nil = default behavior of DB
+    # @return [OccamsRecord::Cursor]
+    #
+    def cursor(name: nil, scroll: nil, hold: nil)
+      Cursor.new(conn, @sql,
+        name: name, scroll: scroll, hold: hold,
+        use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders,
+      )
+    end
+
     private
 
     # Returns the SQL as a String with all variables escaped
@@ -158,45 +228,6 @@ module OccamsRecord
       @sql.match(/\s+FROM\s+"?(\w+)"?/i)&.captures&.first
     end
 
-    #
-    # Returns an Enumerator that yields batches of records, of size "of".
-    # The SQL string must include 'LIMIT %{batch_limit} OFFSET %{batch_offset}'.
-    # The bind values will be provided by OccamsRecord.
-    #
-    # @param of [Integer] batch size
-    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
-    # @return [Enumerator] yields batches
-    #
-    def batches(of:, use_transaction: true, append_order_by: nil)
-      unless @sql =~ /LIMIT\s+%\{batch_limit\}/i and @sql =~ /OFFSET\s+%\{batch_offset\}/i
-        raise ArgumentError, "When using find_each/find_in_batches you must specify 'LIMIT %{batch_limit} OFFSET %{batch_offset}'. SQL statement: #{@sql}"
-      end
-
-      Enumerator.new do |y|
-        if use_transaction and conn.open_transactions == 0
-          conn.transaction {
-            run_batches y, of
-          }
-        else
-          run_batches y, of
-        end
-      end
-    end
-
-    def run_batches(y, of)
-      offset = 0
-      loop do
-        results = RawQuery.new(@sql, @binds.merge({
-          batch_limit: of,
-          batch_offset: offset,
-        }), use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders).run
-
-        y.yield results if results.any?
-        break if results.size < of
-        offset += results.size
-      end
-    end
-
     def conn
       @conn ||= @eager_loaders.model&.connection || ActiveRecord::Base.connection
     end

data/lib/occams-record/version.rb

@@ -2,6 +2,6 @@
 # Main entry point for using OccamsRecord.
 #
 module OccamsRecord
-  # Library version
-  VERSION = "1.3.0".freeze
+  # @private
+  VERSION = "1.4.0-beta1".freeze
 end

data/lib/occams-record.rb

@@ -5,9 +5,15 @@ require 'occams-record/measureable'
 require 'occams-record/eager_loaders/eager_loaders'
 require 'occams-record/results/results'
 require 'occams-record/results/row'
+require 'occams-record/cursor'
+require 'occams-record/errors'
+
+require 'occams-record/batches/offset_limit/scoped'
+require 'occams-record/batches/offset_limit/raw_query'
+require 'occams-record/batches/cursor_helpers'
+
 require 'occams-record/query'
 require 'occams-record/raw_query'
-require 'occams-record/errors'
 
 module OccamsRecord
   autoload :Ugly, 'occams-record/ugly'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: occams-record
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0.pre.beta1
 platform: ruby
 authors:
 - Jordan Hollinger
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-30 00:00:00.000000000 Z
+date: 2022-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -44,8 +44,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: A faster, lower-memory querying API for ActiveRecord that returns results
-  as unadorned, read-only objects.
+description: A faster, lower-memory, fuller-featured querying API for ActiveRecord
+  that returns results as unadorned, read-only objects.
 email: jordan.hollinger@gmail.com
 executables: []
 extensions: []
@@ -53,7 +53,10 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/occams-record.rb
-- lib/occams-record/batches.rb
+- lib/occams-record/batches/cursor_helpers.rb
+- lib/occams-record/batches/offset_limit/raw_query.rb
+- lib/occams-record/batches/offset_limit/scoped.rb
+- lib/occams-record/cursor.rb
 - lib/occams-record/eager_loaders/ad_hoc_base.rb
 - lib/occams-record/eager_loaders/ad_hoc_many.rb
 - lib/occams-record/eager_loaders/ad_hoc_one.rb
@@ -91,9 +94,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.1.6
 signing_key:

data/lib/occams-record/batches.rb

@@ -1,113 +0,0 @@
-module OccamsRecord
-  #
-  # Methods for building batch finding methods. It expects "model" and "scope" methods to be present.
-  #
-  module Batches
-    #
-    # Load records in batches of N and yield each record to a block if given. If no block is given,
-    # returns an Enumerator.
-    #
-    # NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
-    # to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
-    # of a transaction.
-    #
-    # @param batch_size [Integer]
-    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
-    # @param append_order_by [String] Append this column to ORDER BY to ensure consistent results. Defaults to the primary key. Pass false to disable.
-    # @yield [OccamsRecord::Results::Row]
-    # @return [Enumerator] will yield each record
-    #
-    def find_each(batch_size: 1000, use_transaction: true, append_order_by: nil)
-      enum = Enumerator.new { |y|
-        batches(of: batch_size, use_transaction: use_transaction, append_order_by: append_order_by).each { |batch|
-          batch.each { |record| y.yield record }
-        }
-      }
-      if block_given?
-        enum.each { |record| yield record }
-      else
-        enum
-      end
-    end
-
-    #
-    # Load records in batches of N and yield each batch to a block if given.
-    # If no block is given, returns an Enumerator.
-    #
-    # NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
-    # to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
-    # of a transaction.
-    #
-    # @param batch_size [Integer]
-    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
-    # @param append_order_by [String] Append this column to ORDER BY to ensure consistent results. Defaults to the primary key. Pass false to disable.
-    # @yield [OccamsRecord::Results::Row]
-    # @return [Enumerator] will yield each batch
-    #
-    def find_in_batches(batch_size: 1000, use_transaction: true, append_order_by: nil)
-      enum = batches(of: batch_size, use_transaction: use_transaction, append_order_by: append_order_by)
-      if block_given?
-        enum.each { |batch| yield batch }
-      else
-        enum
-      end
-    end
-
-    private
-
-    #
-    # Returns an Enumerator that yields batches of records, of size "of".
-    # NOTE ActiveRecord 5+ provides the 'in_batches' method to do something
-    # similiar, although 4.2 does not. Also it does not respect ORDER BY,
-    # whereas this does.
-    #
-    # @param of [Integer] batch size
-    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
-    # @param append_order_by [String] Append this column to ORDER BY to ensure consistent results. Defaults to the primary key. Pass false to disable.
-    # @return [Enumerator] yields batches
-    #
-    def batches(of:, use_transaction: true, append_order_by: nil)
-      append_order =
-        case append_order_by
-        when false then nil
-        when nil then model.primary_key
-        else append_order_by
-        end
-
-      Enumerator.new do |y|
-        if use_transaction and model.connection.open_transactions == 0
-          model.connection.transaction {
-            run_batches y, of, append_order
-          }
-        else
-          run_batches y, of, append_order
-        end
-      end
-    end
-
-    def run_batches(y, of, append_order_by = nil)
-      limit = scope.limit_value
-      batch_size = limit && limit < of ? limit : of
-
-      offset = scope.offset_value || 0
-      out_of_records, count = false, 0
-      order_by =
-        if append_order_by
-          append_order_by.to_s == model.primary_key.to_s ? append_order_by.to_sym : append_order_by
-        end
-
-      until out_of_records
-        l = limit && batch_size > limit - count ? limit - count : batch_size
-        q = scope
-        q = q.order(order_by) if order_by
-        q = q.offset(offset).limit(l)
-        results = Query.new(q, use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders).run
-
-        y.yield results if results.any?
-        count += results.size
-        offset += results.size
-        out_of_records = results.size < batch_size || (limit && count >= limit)
-      end
-    end
-  end
-end