occams-record 1.0.0.rc9 → 1.0.0.rc10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dffcac805f1cbe701d082c2b965aeaa98c89c289fbb1d5bf3e164e5ce5ad8af1
-  data.tar.gz: a94cda603a6c408c77f751e2d9358a1f52decb34464bbd8967af9f16fd978712
+  metadata.gz: c0cdafe04c023de8a937ebef5fb1c274459e11b10c06f7fec38cf5632b93c969
+  data.tar.gz: f98a82b06655ddaf71bcc68f171b3aa077b08f64bdfca9f35725e8fe2ad0c4f3
 SHA512:
-  metadata.gz: afeb27a07b31d12257161618ea4d9a17088963014d354691de035b3e0e423d7f8d5d1c62991f4c9256282ec7ab4afecf32806ba4f56182cdb9d36a91e4436bf0
-  data.tar.gz: aecefa1ddbcb7179b5b24805d1ffcadbd71a25a1f4888c8bb89a433f7900f39dd576885a4e85e33502f568cef82b9da04eccdcaa3f4d9c44964ac59631595442
+  metadata.gz: dfe0c1a6f0c18be9e37c17b5928a4911e62615cc95e5399b3d181b75d769bdf803c13823773d285daab4602f526c397d15f3d0f9dfa62fa9970539e07028a9e7
+  data.tar.gz: e606aee2bae1577a3d77649979efa76c32b30b40614d0bbd4acb5f19ea1f453c16175e466a9afbad6997112c1cb12a64b6b890dd929cafbee039d2e1cbdb665d

data/README.md

@@ -55,9 +55,9 @@ orders = OccamsRecord.
   run
 ````
 
-`each`, `map`, `reduce`, and other Enumerable methods may be used instead of *run*. `find_each` and `find_in_batches` are also supported. Occams Record has great support for raw SQL queries too, but we'll get to those later.
+`each`, `map`, `reduce`, and other Enumerable methods may be used instead of *run*. `find_each` and `find_in_batches` are also supported, and unlike in ActiveRecord, `ORDER BY` works as you'd expect.
 
-NOTE Unlike in Active Record, `find_each` and `find_in_batches` respect *ORDER BY*. However, to ensure consistency, it's strongly recomended to use `find_each`/`find_in_batches` inside of a transaction.
+Occams Record has great support for raw SQL queries too, but we'll get to those later.
 
 ## Basic eager loading
 

data/lib/occams-record/batches.rb

@@ -4,22 +4,21 @@ module OccamsRecord
   #
   module Batches
     #
-    # Load records in batches of N and yield each record to a block if given.
-    # If no block is given, returns an Enumerator.
+    # Load records in batches of N and yield each record to a block if given. If no block is given,
+    # returns an Enumerator.
     #
-    # NOTE Unlike ActiveRecord's find_each, order is respected. The primary key will be appended
-    # to the ORDER BY clause to help ensure consistent batches. HOWEVER, it's still possible for
-    # batches to be "corrupted" (miss records or repeat records) if table data changes out from
-    # enderneath them. To prevent this, it's strongly recomended to always run find_each inside
+    # NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
+    # to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
     # of a transaction.
     #
     # @param batch_size [Integer]
+    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
     # @yield [OccamsRecord::Results::Row]
     # @return [Enumerator] will yield each record
     #
-    def find_each(batch_size: 1000)
+    def find_each(batch_size: 1000, use_transaction: true)
       enum = Enumerator.new { |y|
-        batches(of: batch_size).each { |batch|
+        batches(of: batch_size, use_transaction: use_transaction).each { |batch|
           batch.each { |record| y.yield record }
         }
       }
@@ -34,18 +33,17 @@ module OccamsRecord
     # Load records in batches of N and yield each batch to a block if given.
     # If no block is given, returns an Enumerator.
     #
-    # NOTE Unlike ActiveRecord's find_in_batches, order is respected. The primary key will be appended
-    # to the ORDER BY clause to help ensure consistent batches. HOWEVER, it's still possible for
-    # batches to be "corrupted" (miss records or repeat records) if table data changes out from
-    # enderneath them. To prevent this, it's strongly recomended to always run find_in_batches inside
+    # NOTE Unlike ActiveRecord's find_each, ORDER BY is respected. The primary key will be appended
+    # to the ORDER BY clause to help ensure consistent batches. Additionally, it will be run inside
     # of a transaction.
     #
     # @param batch_size [Integer]
+    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
     # @yield [OccamsRecord::Results::Row]
     # @return [Enumerator] will yield each batch
     #
-    def find_in_batches(batch_size: 1000)
-      enum = batches(of: batch_size)
+    def find_in_batches(batch_size: 1000, use_transaction: true)
+      enum = batches(of: batch_size, use_transaction: use_transaction)
       if block_given?
         enum.each { |batch| yield batch }
       else
@@ -62,29 +60,37 @@ module OccamsRecord
     # whereas this does.
     #
     # @param of [Integer] batch size
+    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
     # @return [Enumerator] yields batches
     #
-    def batches(of:)
-      if model.connection.open_transactions == 0
-        $stderr.puts "Occams Record Warning: find_each or find_in_batches is being run outside of transaction. Batch consistency can only be ensured within a transaction."
+    def batches(of:, use_transaction: true)
+      Enumerator.new do |y|
+        if use_transaction and model.connection.open_transactions == 0
+          model.connection.transaction {
+            run_batches y, of
+          }
+        else
+          run_batches y, of
+        end
       end
+    end
 
+    def run_batches(y, of)
       limit = scope.limit_value
       batch_size = limit && limit < of ? limit : of
-      Enumerator.new do |y|
-        offset = scope.offset_value || 0
-        out_of_records, count = false, 0
 
-        until out_of_records
-          l = limit && batch_size > limit - count ? limit - count : batch_size
-          q = scope.order(model.primary_key.to_sym).offset(offset).limit(l)
-          results = Query.new(q, use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders).run
+      offset = scope.offset_value || 0
+      out_of_records, count = false, 0
 
-          y.yield results if results.any?
-          count += results.size
-          offset += results.size
-          out_of_records = results.size < batch_size || (limit && count >= limit)
-        end
+      until out_of_records
+        l = limit && batch_size > limit - count ? limit - count : batch_size
+        q = scope.order(model.primary_key.to_sym).offset(offset).limit(l)
+        results = Query.new(q, use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders).run
+
+        y.yield results if results.any?
+        count += results.size
+        offset += results.size
+        out_of_records = results.size < batch_size || (limit && count >= limit)
       end
     end
   end

data/lib/occams-record/raw_query.rb

@@ -139,26 +139,37 @@ module OccamsRecord
     # The bind values will be provided by OccamsRecord.
     #
     # @param of [Integer] batch size
+    # @param use_transaction [Boolean] Ensure it runs inside of a database transaction
     # @return [Enumerator] yields batches
     #
-    def batches(of:)
+    def batches(of:, use_transaction: true)
       unless @sql =~ /LIMIT\s+%\{batch_limit\}/i and @sql =~ /OFFSET\s+%\{batch_offset\}/i
         raise ArgumentError, "When using find_each/find_in_batches you must specify 'LIMIT %{batch_limit} OFFSET %{batch_offset}'. SQL statement: #{@sql}"
       end
 
       Enumerator.new do |y|
-        offset = 0
-        loop do
-          results = RawQuery.new(@sql, @binds.merge({
-            batch_limit: of,
-            batch_offset: offset,
-          }), use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders).run
-
-          y.yield results if results.any?
-          break if results.size < of
-          offset += results.size
+        if use_transaction and @conn.open_transactions == 0
+          @conn.transaction {
+            run_batches y, of
+          }
+        else
+          run_batches y, of
         end
       end
     end
+
+    def run_batches(y, of)
+      offset = 0
+      loop do
+        results = RawQuery.new(@sql, @binds.merge({
+          batch_limit: of,
+          batch_offset: offset,
+        }), use: @use, query_logger: @query_logger, eager_loaders: @eager_loaders).run
+
+        y.yield results if results.any?
+        break if results.size < of
+        offset += results.size
+      end
+    end
   end
 end

data/lib/occams-record/version.rb

@@ -3,5 +3,5 @@
 #
 module OccamsRecord
   # Library version
-  VERSION = '1.0.0.rc9'.freeze
+  VERSION = '1.0.0.rc10'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: occams-record
 version: !ruby/object:Gem::Version
-  version: 1.0.0.rc9
+  version: 1.0.0.rc10
 platform: ruby
 authors:
 - Jordan Hollinger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-22 00:00:00.000000000 Z
+date: 2019-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord