occams-record 1.3.0.rc1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd827ce90c189d89558863ece800e233c3296f590f18e6b756d4f4983c673e7e
-  data.tar.gz: 0bfd98422a9e2a313aae5faef338e442979235377ba1ddad9f5b38e2ccee1b27
+  metadata.gz: c1dc718c0f99f5ad039974f8db3bd1c5304e41a3b3275d26dd1d86235c7790fe
+  data.tar.gz: 8af374148ca1f1c8342b7eb8fdb30031d6c31a1d923a6e7cab55b96712ed2602
 SHA512:
-  metadata.gz: 00352ecd359c2cd617f29d225aa3a566eed578a81f875714896b0e380931c9e5611b87971f2e68197225a59ebab5e7cefb96f65f4855f08481ca38b02d9f9c8c
-  data.tar.gz: 6f82a98e4790c92d86d44f4ea2e4babd8dde300a08a60106d8e1f8b0e1e0210351ff89915d1f56e44160779b358df8e8d4355ab69f14f81ee23d91e13f033f71
+  metadata.gz: 54b539bc70b1fc92c087fbc91f7edbaa1eb4dabacfda80ca0afa4ccd4ac53e832d431f3c95e35785cfa7bd7075879c655ae54019c3f341255b4411929b5ae615
+  data.tar.gz: c1c9aed1007bcfeb0e577d08af6f86fa340e375fede08af03bd18fbc2538cfcbd4838fc5bd07c672fca6e02ca3c0ec9e8d442b3e3275deff58215dfba873df7a

data/README.md

@@ -9,6 +9,7 @@ OccamsRecord is a high-efficiency, advanced query library for use alongside Acti
 * 3x-5x faster than ActiveRecord queries, *minimum*.
 * Uses 1/3 the memory of ActiveRecord query results.
 * Eliminates the N+1 query problem. (This often exceeds the baseline 3x-5x gain.)
+* Support for cursors (Postgres only, new in v1.4.0-beta1)
 
 ### 2) Supercharged querying & eager loading
 
@@ -32,6 +33,16 @@ OccamsRecord
   }
 ```
 
+**Use cursors**
+
+```ruby
+OccamsRecord
+  .query(Order.order("created_at DESC"))
+  .find_each_with_cursor { |order|
+    ...
+  }
+```
+
 **Use `find_each`/`find_in_batches` with raw SQL**
 
 ```ruby
@@ -156,15 +167,59 @@ orders = OccamsRecord
   .run
 ```
 
-Occams Record also supports loading ad hoc associations using raw SQL. We'll get to that in the next section.
+Occams Record also supports loading ad hoc associations using raw SQL. We'll get to that in a later section.
+
+## Query with cursors
+
+`find_each_with_cursor`/`find_in_batches_with_cursor` work like `find_each`/`find_in_batches`, except they use cursors. For large data sets, cursors offer a noticible speed boost. Postgres only.
+
+```ruby
+OccamsRecord
+  .query(q)
+  .eager_load(:customer)
+  .find_each_with_cursor do |order|
+    ...
+  end
+```
+
+The `cursor.open` method allows lower level access to cursor behavior. See `OccamsRecord::Cursor` for more info.
+
+```ruby
+orders = OccamsRecord
+  .query(q)
+  .eager_load(:customer)
+  .cursor
+  .open do |cursor|
+    cursor.move(:forward, 300)
+    cursor.fetch(:forward, 100)
+  end
+```
 
 ## Raw SQL queries
 
 ActiveRecord has raw SQL escape hatches like `find_by_sql` and `exec_query`, but they give up critical features like eager loading and `find_each`/`find_in_batches`. Occams Record's escape hatches don't make you give up anything.
 
-**Batched loading**
+**Batched loading with cursors**
 
-To use `find_each`/`find_in_batches` you must provide the limit and offset statements yourself; Occams will provide the values. Also, notice that the binding syntax is a bit different (it uses Ruby's built-in named string substitution).
+`find_each_with_cursor`, `find_in_batches_with_cursor`, and `cursor.open` are all available.
+
+```ruby
+OccamsRecord
+  .sql("
+    SELECT * FROM orders
+    WHERE order_date > %{date}
+    ORDER BY order_date DESC, id
+  ", {
+    date: 10.years.ago
+  })
+  .find_each_with_cursor(batch_size: 1000) do |order|
+    ...
+  end
+```
+
+**Batched loading without cursors**
+
+If your database doesn't support cursors, you can use `find_each`/`find_in_batches`. Just provide `LIMIT` and `OFFSET` (see below), and Occams will plug in the right numbers.
 
 ```ruby
 OccamsRecord
@@ -300,6 +355,9 @@ bundle exec rake test
 
 # test against Postgres
 TEST_DATABASE_URL=postgres://postgres@localhost:5432/occams_record bundle exec rake test
+
+# test against MySQL
+TEST_DATABASE_URL=mysql2://root:@127.0.0.1:3306/occams_record bundle exec rake test
 ```
 
 **Test against all supported ActiveRecord versions**
@@ -314,7 +372,7 @@ bundle exec appraisal rake test
 bundle exec appraisal ar-7.0 rake test
 
 # test against Postgres
-TEST_DATABASE_URL=postgres://postgres@localhost:5432/occams_record bundle exec appraisal rake test
+TEST_DATABASE_URL=postgresql://postgres@localhost:5432/occams_record bundle exec appraisal rake test
 ```
 
 # License

data/lib/occams-record/batches/cursor_helpers.rb

@@ -0,0 +1,59 @@
+module OccamsRecord
+  module Batches
+    module CursorHelpers
+      #
+      # Loads records in batches of N and yields each record to a block (if given). If no block is given,
+      # returns an Enumerator.
+      #
+      # NOTE Unlike find_each, batches are loaded using a cursor, which offers better performance.
+      # Postgres only. See the docs for OccamsRecord::Cursor for more details.
+      #
+      # @param batch_size [Integer] fetch this many rows at once
+      # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+      # @yield [OccamsRecord::Results::Row]
+      # @return [Enumerator] will yield each record
+      #
+      def find_each_with_cursor(batch_size: 1000, use_transaction: true)
+        enum = Enumerator.new { |y|
+          cursor.open(use_transaction: use_transaction) { |c|
+            c.each(batch_size: batch_size) { |record|
+              y.yield record
+            }
+          }
+        }
+        if block_given?
+          enum.each { |record| yield record }
+        else
+          enum
+        end
+      end
+
+      #
+      # Loads records in batches of N and yields each batch to a block (if given). If no block is given,
+      # returns an Enumerator.
+      #
+      # NOTE Unlike find_in_batches, batches are loaded using a cursor, which offers better performance.
+      # Postgres only. See the docs for OccamsRecord::Cursor for more details.
+      #
+      # @param batch_size [Integer] fetch this many rows at once
+      # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+      # @yield [OccamsRecord::Results::Row]
+      # @return [Enumerator] will yield each record
+      #
+      def find_in_batches_with_cursor(batch_size: 1000, use_transaction: true)
+        enum = Enumerator.new { |y|
+          cursor.open(use_transaction: use_transaction) { |c|
+            c.each_batch(batch_size: batch_size) { |batch|
+              y.yield batch
+            }
+          }
+        } 
+        if block_given?
+          enum.each { |batch| yield batch }
+        else
+          enum
+        end
+      end
+    end
+  end
+end

data/lib/occams-record/batches/offset_limit/raw_query.rb

@@ -0,0 +1,56 @@
+module OccamsRecord
+  module Batches
+    module OffsetLimit
+      #
+      # Implements batched loading for pure SQL.
+      #
+      class RawQuery
+        def initialize(conn, sql, binds, use: nil, query_logger: nil, eager_loaders: nil)
+          @conn, @sql, @binds = conn, sql, binds
+          @use, @query_logger, @eager_loaders = use, query_logger, eager_loaders
+
+          unless @sql =~ /LIMIT\s+%\{batch_limit\}/i and @sql =~ /OFFSET\s+%\{batch_offset\}/i
+            raise ArgumentError, "When using find_each/find_in_batches you must specify 'LIMIT %{batch_limit} OFFSET %{batch_offset}'. SQL statement: #{@sql}"
+          end
+        end
+
+        #
+        # Returns an Enumerator that yields batches of records, of size "of".
+        # The SQL string must include 'LIMIT %{batch_limit} OFFSET %{batch_offset}'.
+        # The bind values will be provided by OccamsRecord.
+        #
+        # @param batch_size [Integer] batch size
+        # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+        # @return [Enumerator] yields batches
+        #
+        def enum(batch_size:, use_transaction: true)
+          Enumerator.new do |y|
+            if use_transaction and @conn.open_transactions == 0
+              @conn.transaction {
+                run_batches y, batch_size
+              }
+            else
+              run_batches y, batch_size
+            end
+          end
+        end
+
+        private
+
+        def run_batches(y, of)
+          offset = 0
+          loop do
+            results = ::OccamsRecord::RawQuery.new(@sql, @binds.merge({
+              batch_limit: of,
+              batch_offset: offset,
+            }), use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders).run
+
+            y.yield results if results.any?
+            break if results.size < of
+            offset += results.size
+          end
+        end
+      end
+    end
+  end
+end

data/lib/occams-record/batches/offset_limit/scoped.rb

@@ -0,0 +1,72 @@
+module OccamsRecord
+  module Batches
+    module OffsetLimit
+      #
+      # Implements batched loading for ActiveRecord model scopes.
+      #
+      class Scoped
+        def initialize(model, scope, use: nil, query_logger: nil, eager_loaders: nil)
+          @model, @scope = model, scope
+          @use, @query_logger, @eager_loaders = use, query_logger, eager_loaders
+        end
+
+        #
+        # Returns an Enumerator that yields batches of records, of size "of".
+        # NOTE ActiveRecord 5+ provides the 'in_batches' method to do something
+        # similiar, although 4.2 does not. Also it does not respect ORDER BY,
+        # whereas this does.
+        #
+        # @param batch_size [Integer] batch size
+        # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+        # @param append_order_by [String] Append this column to ORDER BY to ensure consistent results. Defaults to the primary key. Pass false to disable.
+        # @return [Enumerator] yields batches
+        #
+        def enum(batch_size:, use_transaction: true, append_order_by: nil)
+          append_order =
+            case append_order_by
+            when false then nil
+            when nil then @model.primary_key
+            else append_order_by
+            end
+
+          Enumerator.new do |y|
+            if use_transaction and @model.connection.open_transactions == 0
+              @model.connection.transaction {
+                run_batches y, batch_size, append_order
+              }
+            else
+              run_batches y, batch_size, append_order
+            end
+          end
+        end
+
+        private
+
+        def run_batches(y, of, append_order_by = nil)
+          limit = @scope.limit_value
+          batch_size = limit && limit < of ? limit : of
+
+          offset = @scope.offset_value || 0
+          out_of_records, count = false, 0
+          order_by =
+            if append_order_by
+              append_order_by.to_s == @model.primary_key.to_s ? append_order_by.to_sym : append_order_by
+            end
+
+          until out_of_records
+            l = limit && batch_size > limit - count ? limit - count : batch_size
+            q = @scope
+            q = q.order(order_by) if order_by
+            q = q.offset(offset).limit(l)
+            results = ::OccamsRecord::Query.new(q, use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders).run
+
+            y.yield results if results.any?
+            count += results.size
+            offset += results.size
+            out_of_records = results.size < batch_size || (limit && count >= limit)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/occams-record/cursor.rb

@@ -0,0 +1,237 @@
+require 'securerandom'
+
+module OccamsRecord
+  #
+  # An interface to database cursors. Supported databases:
+  #   * PostgreSQL
+  #
+  class Cursor
+    # @private
+    SCROLL = {
+      true => "SCROLL",
+      false => "NO SCROLL",
+      nil => "",
+    }.freeze
+
+    # @private
+    HOLD = {
+      true => "WITH HOLD",
+      false => "WITHOUT HOLD",
+      nil => "",
+    }.freeze
+
+    # @private
+    DIRECTIONS = {
+      next: "NEXT",
+      prior: "PRIOR",
+      first: "FIRST",
+      last: "LAST",
+      absolute: "ABSOLUTE",
+      relative: "RELATIVE",
+      forward: "FORWARD",
+      backward: "BACKWARD",
+    }.freeze
+
+    # @return [ActiveRecord::Connection]
+    attr_reader :conn
+
+    # Name of the cursor
+    # @return [String]
+    attr_reader :name
+
+    # Name of the cursor (safely SQL-escaped)
+    # @return [String]
+    attr_reader :quoted_name
+
+    #
+    # Initializes a new Cursor. NOTE all operations must be performed within a block passed to #open.
+    #
+    # While you CAN manually initialize a cursor, it's more common to get one via OccamsRecord::Query#cursor
+    # or OccamsRecord::RawQuery#cursor.
+    #
+    # @param conn [ActiveRecord::Connection]
+    # @param sql [String] The query to run
+    # @param name [String] Specify a name for the cursor (defaults to a random name)
+    # @param scroll [Boolean] true = SCROLL, false = NO SCROLL, nil = default behavior of DB
+    # @param hold [Boolean] true = WITH HOLD, false = WITHOUT HOLD, nil = default behavior of DB
+    # @param use [Array<Module>] optional Module to include in the result class (single or array)
+    # @param query_logger [Array] (optional) an array into which all queries will be inserted for logging/debug purposes
+    # @param eager_loaders [OccamsRecord::EagerLoaders::Context]
+    #
+    def initialize(conn, sql, name: nil, scroll: nil, hold: nil, use: nil, query_logger: nil, eager_loaders: nil)
+      @conn, @sql = conn, sql
+      @scroll = SCROLL.fetch(scroll)
+      @hold = HOLD.fetch(hold)
+      @use, @query_logger, @eager_loaders = use, query_logger, eager_loaders
+      @name = name || "occams_cursor_#{SecureRandom.hex 4}"
+      @quoted_name = conn.quote_table_name(@name)
+    end
+
+    #
+    # Declares and opens the cursor, runs the given block (yielding self), and closes the cursor.
+    #
+    #   cursor.open do |c|
+    #     c.fetch :forward, 100
+    #   end
+    #
+    # @param use_transaction [Boolean] When true, ensures it's wrapped in a transaction
+    # @yield [self]
+    # @return the value returned by the block
+    #
+    def open(use_transaction: true)
+      raise ArgumentError, "A block is required" unless block_given?
+      if use_transaction and conn.open_transactions == 0
+        conn.transaction {
+          perform { yield self }
+        }
+      else
+        perform { yield self }
+      end
+    end
+
+    #
+    # Loads records in batches of N and yields each record to a block (if given). If no block is given,
+    # returns an Enumerator.
+    #
+    #   cursor.open do |c|
+    #     c.each do |record|
+    #       ...
+    #     end
+    #   end
+    #
+    # @param batch_size [Integer] fetch this many rows at once
+    #
+    def each(batch_size: 1000)
+      enum = Enumerator.new { |y|
+        each_batch(batch_size: batch_size).each { |batch|
+          batch.each { |record| y.yield record }
+        }
+      }
+      if block_given?
+        enum.each { |record| yield record }
+      else
+        enum
+      end
+    end
+
+    #
+    # Loads records in batches of N and yields each batch to a block (if given). If no block is given,
+    # returns an Enumerator.
+    #
+    #   cursor.open do |c|
+    #     c.each_batch do |batch|
+    #       ...
+    #     end
+    #   end
+    #
+    # @param batch_size [Integer] fetch this many rows at once
+    #
+    def each_batch(batch_size: 1000)
+      enum = Enumerator.new { |y|
+        out_of_records = false
+        until out_of_records
+          results = fetch :forward, batch_size
+          y.yield results if results.any?
+          out_of_records = results.size < batch_size
+        end
+      }
+      if block_given?
+        enum.each { |batch| yield batch }
+      else
+        enum
+      end
+    end
+
+    #
+    # Fetch records in the given direction.
+    #
+    #   cursor.open do |c|
+    #     c.fetch :forward, 100
+    #     ...
+    #   end
+    #
+    # @param direction [Symbol] :next, :prior, :first, :last, :absolute, :relative, :forward or :backward
+    # @param num [Integer] number of rows to fetch (optional for some directions)
+    # @return [OccamsRecord::Results::Row]
+    #
+    def fetch(direction, num = nil)
+      query "FETCH %{dir} %{num} FROM %{name}".freeze % {
+        dir: DIRECTIONS.fetch(direction),
+        num: num&.to_i,
+        name: @quoted_name,
+      }
+    end
+
+    #
+    # Move the cursor the given direction.
+    #
+    #   cursor.open do |c|
+    #     ...
+    #     c.move :backward, 100
+    #     ...
+    #   end
+    #
+    # @param direction [Symbol] :next, :prior, :first, :last, :absolute, :relative, :forward or :backward
+    # @param num [Integer] number of rows to move (optional for some directions)
+    #
+    def move(direction, num = nil)
+      query "MOVE %{dir} %{num} FROM %{name}".freeze % {
+        dir: DIRECTIONS.fetch(direction),
+        num: num&.to_i,
+        name: @quoted_name,
+      }
+    end
+
+    #
+    # Run an arbitrary query on the cursor. Use 'binds' to escape inputs.
+    #
+    #   cursor.open do |c|
+    #     c.query("FETCH FORWARD %{num} FOR #{c.quoted_name}", {num: 100})
+    #     ...
+    #   end
+    #
+    def query(sql, binds = {})
+      ::OccamsRecord::RawQuery.new(sql, binds, use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders, connection: conn).run
+    end
+
+    #
+    # Run an arbitrary command on the cursor. Use 'binds' to escape inputs.
+    #
+    #   cursor.open do |c|
+    #     c.execute("MOVE FORWARD %{num} FOR #{c.quoted_name}", {num: 100})
+    #     ...
+    #   end
+    #
+    def execute(sql, binds = {})
+      conn.execute(sql % binds.reduce({}) { |acc, (key, val)|
+        acc[key] = conn.quote(val)
+        acc
+      })
+    end
+
+    private
+
+    def perform
+      ex = nil
+      conn.execute "DECLARE %{name} %{scroll} CURSOR %{hold} FOR %{query}".freeze % {
+        name: @quoted_name,
+        scroll: @scroll,
+        hold: @hold,
+        query: @sql,
+      }
+      yield
+    rescue => e
+      ex = e
+      raise ex
+    ensure
+      begin
+        conn.execute "CLOSE %{name}".freeze % {name: @quoted_name}
+      rescue => e
+        # Don't let an error from CLOSE (like a dead transaction) hide what lead to the error with CLOSE (like bad SQL that raised an error and aborted the transaction)
+        raise ex || e
+      else
+        raise ex if ex
+      end
+    end
+  end
+end

data/lib/occams-record/query.rb

@@ -1,5 +1,3 @@
-require 'occams-record/batches'
-
 module OccamsRecord
   #
   # Starts building a OccamsRecord::Query. Pass it a scope from any of ActiveRecord's query builder
@@ -36,7 +34,7 @@ module OccamsRecord
     # @return [ActiveRecord::Relation] scope for building the main SQL query
     attr_reader :scope
 
-    include Batches
+    include OccamsRecord::Batches::CursorHelpers
     include EagerLoaders::Builder
     include Enumerable
     include Measureable
@@ -154,5 +152,75 @@ module OccamsRecord
         to_a.each
       end
     end
+
+    #
+    # Load records in batches of N and yield each record to a block if given. If no block is given,
+    # returns an Enumerator.
+    #
+    # NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
+    # to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
+    # of a transaction.
+    #
+    # @param batch_size [Integer]
+    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+    # @param append_order_by [String] Append this column to ORDER BY to ensure consistent results. Defaults to the primary key. Pass false to disable.
+    # @yield [OccamsRecord::Results::Row]
+    # @return [Enumerator] will yield each record
+    #
+    def find_each(batch_size: 1000, use_transaction: true, append_order_by: nil)
+      enum = Enumerator.new { |y|
+        find_in_batches(batch_size: 1000, use_transaction: use_transaction, append_order_by: append_order_by).each { |batch|
+          batch.each { |record| y.yield record }
+        }
+      }
+      if block_given?
+        enum.each { |record| yield record }
+      else
+        enum
+      end
+    end
+
+    #
+    # Load records in batches of N and yield each batch to a block if given.
+    # If no block is given, returns an Enumerator.
+    #
+    # NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
+    # to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
+    # of a transaction.
+    #
+    # @param batch_size [Integer]
+    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+    # @param append_order_by [String] Append this column to ORDER BY to ensure consistent results. Defaults to the primary key. Pass false to disable.
+    # @yield [OccamsRecord::Results::Row]
+    # @return [Enumerator] will yield each batch
+    #
+    def find_in_batches(batch_size: 1000, use_transaction: true, append_order_by: nil)
+      enum = Batches::OffsetLimit::Scoped
+        .new(model, scope, use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders)
+        .enum(batch_size: batch_size, use_transaction: use_transaction, append_order_by: append_order_by)
+      if block_given?
+        enum.each { |batch| yield batch }
+      else
+        enum
+      end
+    end
+
+    #
+    # Returns a cursor you can open and perform operations on. A lower-level alternative to 
+    # find_each_with_cursor and find_in_batches_with_cursor.
+    #
+    # NOTE Postgres only. See the docs for OccamsRecord::Cursor for more details.
+    #
+    # @param name [String] Specify a name for the cursor (defaults to a random name)
+    # @param scroll [Boolean] true = SCROLL, false = NO SCROLL, nil = default behavior of DB
+    # @param hold [Boolean] true = WITH HOLD, false = WITHOUT HOLD, nil = default behavior of DB
+    # @return [OccamsRecord::Cursor]
+    #
+    def cursor(name: nil, scroll: nil, hold: nil)
+      Cursor.new(model.connection, scope.to_sql,
+        name: name, scroll: scroll, hold: hold,
+        use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders,
+      )
+    end
   end
 end

data/lib/occams-record/raw_query.rb

@@ -61,7 +61,7 @@ module OccamsRecord
     # @return [Hash]
     attr_reader :binds
 
-    include Batches
+    include OccamsRecord::Batches::CursorHelpers
     include EagerLoaders::Builder
     include Enumerable
     include Measureable
@@ -75,13 +75,15 @@ module OccamsRecord
     # @param eager_loaders [OccamsRecord::EagerLoaders::Context]
     # @param query_logger [Array] (optional) an array into which all queries will be inserted for logging/debug purposes
     # @param measurements [Array]
+    # @param connection
     #
-    def initialize(sql, binds, use: nil, eager_loaders: nil, query_logger: nil, measurements: nil)
+    def initialize(sql, binds, use: nil, eager_loaders: nil, query_logger: nil, measurements: nil, connection: nil)
       @sql = sql
       @binds = binds
       @use = use
       @eager_loaders = eager_loaders || EagerLoaders::Context.new
       @query_logger, @measurements = query_logger, measurements
+      @conn = connection
     end
 
     #
@@ -139,6 +141,74 @@ module OccamsRecord
       end
     end
 
+    #
+    # Load records in batches of N and yield each record to a block if given. If no block is given,
+    # returns an Enumerator.
+    #
+    # NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
+    # to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
+    # of a transaction.
+    #
+    # @param batch_size [Integer]
+    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+    # @yield [OccamsRecord::Results::Row]
+    # @return [Enumerator] will yield each record
+    #
+    def find_each(batch_size: 1000, use_transaction: true)
+      enum = Enumerator.new { |y|
+        find_in_batches(batch_size: batch_size, use_transaction: use_transaction).each { |batch|
+          batch.each { |record| y.yield record }
+        }
+      }
+      if block_given?
+        enum.each { |record| yield record }
+      else
+        enum
+      end
+    end
+
+    #
+    # Load records in batches of N and yield each batch to a block if given.
+    # If no block is given, returns an Enumerator.
+    #
+    # NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
+    # to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
+    # of a transaction.
+    #
+    # @param batch_size [Integer]
+    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
+    # @yield [OccamsRecord::Results::Row]
+    # @return [Enumerator] will yield each batch
+    #
+    def find_in_batches(batch_size: 1000, use_transaction: true)
+      enum = Batches::OffsetLimit::RawQuery
+        .new(conn, @sql, @binds, use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders)
+        .enum(batch_size: batch_size, use_transaction: use_transaction)
+      if block_given?
+        enum.each { |batch| yield batch }
+      else
+        enum
+      end
+    end
+
+    #
+    # Returns a cursor you can open and perform operations on. A lower-level alternative to 
+    # find_each_with_cursor and find_in_batches_with_cursor.
+    #
+    # NOTE Postgres only. See the docs for OccamsRecord::Cursor for more details.
+    #
+    # @param name [String] Specify a name for the cursor (defaults to a random name)
+    # @param scroll [Boolean] true = SCROLL, false = NO SCROLL, nil = default behavior of DB
+    # @param hold [Boolean] true = WITH HOLD, false = WITHOUT HOLD, nil = default behavior of DB
+    # @return [OccamsRecord::Cursor]
+    #
+    def cursor(name: nil, scroll: nil, hold: nil)
+      Cursor.new(conn, @sql,
+        name: name, scroll: scroll, hold: hold,
+        use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders,
+      )
+    end
+
     private
 
     # Returns the SQL as a String with all variables escaped
@@ -158,45 +228,6 @@ module OccamsRecord
       @sql.match(/\s+FROM\s+"?(\w+)"?/i)&.captures&.first
     end
 
-    #
-    # Returns an Enumerator that yields batches of records, of size "of".
-    # The SQL string must include 'LIMIT %{batch_limit} OFFSET %{batch_offset}'.
-    # The bind values will be provided by OccamsRecord.
-    #
-    # @param of [Integer] batch size
-    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
-    # @return [Enumerator] yields batches
-    #
-    def batches(of:, use_transaction: true, append_order_by: nil)
-      unless @sql =~ /LIMIT\s+%\{batch_limit\}/i and @sql =~ /OFFSET\s+%\{batch_offset\}/i
-        raise ArgumentError, "When using find_each/find_in_batches you must specify 'LIMIT %{batch_limit} OFFSET %{batch_offset}'. SQL statement: #{@sql}"
-      end
-
-      Enumerator.new do |y|
-        if use_transaction and conn.open_transactions == 0
-          conn.transaction {
-            run_batches y, of
-          }
-        else
-          run_batches y, of
-        end
-      end
-    end
-
-    def run_batches(y, of)
-      offset = 0
-      loop do
-        results = RawQuery.new(@sql, @binds.merge({
-          batch_limit: of,
-          batch_offset: offset,
-        }), use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders).run
-
-        y.yield results if results.any?
-        break if results.size < of
-        offset += results.size
-      end
-    end
-
     def conn
       @conn ||= @eager_loaders.model&.connection || ActiveRecord::Base.connection
     end

data/lib/occams-record/version.rb

@@ -2,6 +2,6 @@
 # Main entry point for using OccamsRecord.
 #
 module OccamsRecord
-  # Library version
-  VERSION = "1.3.0.rc1".freeze
+  # @private
+  VERSION = "1.4.0".freeze
 end

data/lib/occams-record.rb

@@ -5,9 +5,15 @@ require 'occams-record/measureable'
 require 'occams-record/eager_loaders/eager_loaders'
 require 'occams-record/results/results'
 require 'occams-record/results/row'
+require 'occams-record/cursor'
+require 'occams-record/errors'
+
+require 'occams-record/batches/offset_limit/scoped'
+require 'occams-record/batches/offset_limit/raw_query'
+require 'occams-record/batches/cursor_helpers'
+
 require 'occams-record/query'
 require 'occams-record/raw_query'
-require 'occams-record/errors'
 
 module OccamsRecord
   autoload :Ugly, 'occams-record/ugly'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: occams-record
 version: !ruby/object:Gem::Version
-  version: 1.3.0.rc1
+  version: 1.4.0
 platform: ruby
 authors:
 - Jordan Hollinger
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-22 00:00:00.000000000 Z
+date: 2022-05-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -44,8 +44,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: A faster, lower-memory querying API for ActiveRecord that returns results
-  as unadorned, read-only objects.
+description: A faster, lower-memory, fuller-featured querying API for ActiveRecord
+  that returns results as unadorned, read-only objects.
 email: jordan.hollinger@gmail.com
 executables: []
 extensions: []
@@ -53,7 +53,10 @@ extra_rdoc_files: []
 files:
 - README.md
 - lib/occams-record.rb
-- lib/occams-record/batches.rb
+- lib/occams-record/batches/cursor_helpers.rb
+- lib/occams-record/batches/offset_limit/raw_query.rb
+- lib/occams-record/batches/offset_limit/scoped.rb
+- lib/occams-record/cursor.rb
 - lib/occams-record/eager_loaders/ad_hoc_base.rb
 - lib/occams-record/eager_loaders/ad_hoc_many.rb
 - lib/occams-record/eager_loaders/ad_hoc_one.rb
@@ -80,7 +83,7 @@ homepage: https://jhollinger.github.io/occams-record/
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -91,12 +94,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.1.6
-signing_key:
+rubygems_version: 3.0.3.1
+signing_key: 
 specification_version: 4
 summary: The missing high-efficiency query API for ActiveRecord
 test_files: []

data/lib/occams-record/batches.rb

@@ -1,113 +0,0 @@
-module OccamsRecord
-  #
-  # Methods for building batch finding methods. It expects "model" and "scope" methods to be present.
-  #
-  module Batches
-    #
-    # Load records in batches of N and yield each record to a block if given. If no block is given,
-    # returns an Enumerator.
-    #
-    # NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
-    # to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
-    # of a transaction.
-    #
-    # @param batch_size [Integer]
-    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
-    # @param append_order_by [String] Append this column to ORDER BY to ensure consistent results. Defaults to the primary key. Pass false to disable.
-    # @yield [OccamsRecord::Results::Row]
-    # @return [Enumerator] will yield each record
-    #
-    def find_each(batch_size: 1000, use_transaction: true, append_order_by: nil)
-      enum = Enumerator.new { |y|
-        batches(of: batch_size, use_transaction: use_transaction, append_order_by: append_order_by).each { |batch|
-          batch.each { |record| y.yield record }
-        }
-      }
-      if block_given?
-        enum.each { |record| yield record }
-      else
-        enum
-      end
-    end
-
-    #
-    # Load records in batches of N and yield each batch to a block if given.
-    # If no block is given, returns an Enumerator.
-    #
-    # NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
-    # to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
-    # of a transaction.
-    #
-    # @param batch_size [Integer]
-    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
-    # @param append_order_by [String] Append this column to ORDER BY to ensure consistent results. Defaults to the primary key. Pass false to disable.
-    # @yield [OccamsRecord::Results::Row]
-    # @return [Enumerator] will yield each batch
-    #
-    def find_in_batches(batch_size: 1000, use_transaction: true, append_order_by: nil)
-      enum = batches(of: batch_size, use_transaction: use_transaction, append_order_by: append_order_by)
-      if block_given?
-        enum.each { |batch| yield batch }
-      else
-        enum
-      end
-    end
-
-    private
-
-    #
-    # Returns an Enumerator that yields batches of records, of size "of".
-    # NOTE ActiveRecord 5+ provides the 'in_batches' method to do something
-    # similiar, although 4.2 does not. Also it does not respect ORDER BY,
-    # whereas this does.
-    #
-    # @param of [Integer] batch size
-    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
-    # @param append_order_by [String] Append this column to ORDER BY to ensure consistent results. Defaults to the primary key. Pass false to disable.
-    # @return [Enumerator] yields batches
-    #
-    def batches(of:, use_transaction: true, append_order_by: nil)
-      append_order =
-        case append_order_by
-        when false then nil
-        when nil then model.primary_key
-        else append_order_by
-        end
-
-      Enumerator.new do |y|
-        if use_transaction and model.connection.open_transactions == 0
-          model.connection.transaction {
-            run_batches y, of, append_order
-          }
-        else
-          run_batches y, of, append_order
-        end
-      end
-    end
-
-    def run_batches(y, of, append_order_by = nil)
-      limit = scope.limit_value
-      batch_size = limit && limit < of ? limit : of
-
-      offset = scope.offset_value || 0
-      out_of_records, count = false, 0
-      order_by =
-        if append_order_by
-          append_order_by.to_s == model.primary_key.to_s ? append_order_by.to_sym : append_order_by
-        end
-
-      until out_of_records
-        l = limit && batch_size > limit - count ? limit - count : batch_size
-        q = scope
-        q = q.order(order_by) if order_by
-        q = q.offset(offset).limit(l)
-        results = Query.new(q, use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders).run
-
-        y.yield results if results.any?
-        count += results.size
-        offset += results.size
-        out_of_records = results.size < batch_size || (limit && count >= limit)
-      end
-    end
-  end
-end