ar-ondemand 1.1.3 → 1.1.4

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NWUwZjI4MzczNTc2NTkxNGEyMDRmNjBlZjNjMWU4YmE5Zjk3MGZiZg==
+    OTQ1OGY0YzBjZGU4MjkzZTUxMzEzZTg1YzU2NjBiNzc2YmU1ZGQ5Yg==
   data.tar.gz: !binary |-
-    MjhiYTMxYTgxZGI3MGFmOGQxODc2MTcwZDI5OGQ4MDgwOWM0Y2JhOQ==
+    MjM1YmU4MmQ3YjA5NDhkOWNmYTI5NTkwMTRmMzhlMmYwODRlMTZlZg==
 SHA512:
   metadata.gz: !binary |-
-    Y2E4Zjk0MTQzNjliM2RjOWZiZWUwZTA1ZGI1NTUyMTk5YTcwY2Y2ZTFiMzZi
-    OWZlZGYzYjE2NTA3MjIzN2UyOWM2MDhhNGY0NzFiZGU5NTViYzhiYWMyYWM5
-    NzFkNTg5NTc0ODE0ZGNmNmE2Nzk0NmRmZjBkOTJjODkyNjQ4NWU=
+    ZTRiODA1ZTM1ZDNiMjI2MzY3MjJhYjA3NmFjODVlZDQwZjE5M2NhZjA0NzBh
+    MzhmNTgwMGJkNWMzMTc2YTM1YWZhMjFjYzdmODNiMWMwYzExODE3YjVmODE3
+    NjlkOTQwZGExODZkMDFjODMyMWFkMjE3ODczYTI0MTY5NmJjMmU=
   data.tar.gz: !binary |-
-    ZTU4MTg4NWY0MzU3NWY2NzE3ZTQ0YjM0NWNlMDZiMjM1ODg5NWMxNDk4NTc0
-    ZTI1NGFhOTAwMWE4NDdhMTRhOGM4NjNlZWQ4NDI4MjNhMzc1ZmNlYTYzNmZl
-    ZWFhY2I5ZDE3MzkxMTI4MzcyNjAzOTY4ODY3NGI2MDEyZDZmNGQ=
+    NGQ4NmY3MzUyNmI4NTZlY2JmODZmM2VkZmMyOThkMDVmNjJjM2MwZDBlNjY3
+    MDdiOWI2YjFlOGY3ZTg0MDg2ZDk0MzQzNzBlYWQwMWU5NWY0NDM2MTgyZGU1
+    NGQ1MDcyZjY1ZjMyYTE3OTBjZTVmOGU5ZTg1MzIzZTliYjA5MDQ=

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ar-ondemand (1.1.3)
+    ar-ondemand (1.1.4)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
 

data/lib/ar-ondemand/delete_all_by_pk.rb

@@ -2,58 +2,62 @@ require 'active_support/concern'
 
 module ActiveRecord
   module OnDemand
-    module DeleteAllByPkExtension
+    module DeleteAllByPk
       extend ::ActiveSupport::Concern
 
-      # Use this instead of delete_all to perform a delete using the PK of the table, which prevents a complete table scan that locks it
-      # Based on find_in_batches function
-      def delete_all_by_pk(options = {})
-        relation = self
+      module ClassMethods
+        # Use this instead of delete_all to perform a delete using the PK of the table, which prevents a complete table scan that locks it
+        # Based on find_in_batches function
+        def delete_all_by_pk(options = {})
+          relation = self
+          relation = self.scoped unless respond_to? :arel
 
-        unless arel.orders.blank? && arel.taken.blank?
-          ActiveRecord::Base.logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size")
-        end
+          unless relation.arel.orders.blank? && relation.arel.taken.blank?
+            ::ActiveRecord::Base.logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size")
+          end
 
-        if (finder_options = options.except(:start, :batch_size)).present?
-          raise "You can't specify an order, it's forced to be #{batch_order_delete_all_by_pk}" if options[:order].present?
-          raise "You can't specify a limit, it's forced to be the batch_size"  if options[:limit].present?
+          if (finder_options = options.except(:start, :batch_size)).present?
+            raise "You can't specify an order, it's forced to be #{batch_order_delete_all_by_pk}" if options[:order].present?
+            raise "You can't specify a limit, it's forced to be the batch_size"  if options[:limit].present?
 
-          relation = apply_finder_options(finder_options)
-        end
+            relation = apply_finder_options(finder_options)
+          end
 
-        start = options.delete(:start)
-        batch_size = options.delete(:batch_size)
+          start = options.delete(:start)
+          batch_size = options.delete(:batch_size)
 
-        relation = relation.reorder(batch_order_delete_all_by_pk).limit(batch_size) if batch_size
-        records = query_delete_all_by_pk(start ? relation.where(table[primary_key].gteq(start)) : relation)
-        deleted = 0
+          relation = relation.reorder(batch_order_delete_all_by_pk).limit(batch_size) if batch_size
+          records = query_delete_all_by_pk(start ? relation.where(relation.table[primary_key].gteq(start)) : relation)
+          deleted = 0
 
-        while records.any?
-          records_size = records.size
-          primary_key_offset = records.last
+          while records.any?
+            records_size = records.size
+            primary_key_offset = records.last
 
-          deleted += self.unscoped.where(id: records).delete_all
+            deleted += self.unscoped.where(id: records).delete_all
 
-          break if batch_size.nil? || records_size < batch_size
+            break if batch_size.nil? || records_size < batch_size
 
-          records = query_delete_all_by_pk relation.where(table[primary_key].gt(primary_key_offset))
-        end
+            records = query_delete_all_by_pk relation.where(relation.table[primary_key].gt(primary_key_offset))
+          end
 
-        deleted
-      end
+          deleted
+        end
 
-      private
+        private
 
-      def query_delete_all_by_pk(ar)
-        results = ::ActiveRecord::Base.connection.exec_query ar.select([table[primary_key]]).to_sql
-        results.rows.flatten
-      end
+        def query_delete_all_by_pk(ar)
+          results = ::ActiveRecord::Base.connection.exec_query ar.select([ar.table[primary_key]]).to_sql
+          results.rows.flatten
+        end
 
-      def batch_order_delete_all_by_pk
-        "#{quoted_table_name}.#{quoted_primary_key} ASC"
+        def batch_order_delete_all_by_pk
+          "#{quoted_table_name}.#{quoted_primary_key} ASC"
+        end
       end
     end
   end
 end
 
-::ActiveRecord::Relation.send(:include, ::ActiveRecord::OnDemand::DeleteAllByPkExtension)
+::ActiveRecord::Base.send     :include, ::ActiveRecord::OnDemand::DeleteAllByPk
+::ActiveRecord::Relation.send :include, ::ActiveRecord::OnDemand::DeleteAllByPk::ClassMethods

data/lib/ar-ondemand/for_reading.rb

@@ -4,67 +4,75 @@ require 'ar-ondemand/record'
 
 module ActiveRecord
   module OnDemand
-    module ForReadingExtension
+    module Reading
       extend ::ActiveSupport::Concern
 
-      # Ripped from the find_in_batches function, but customized to return an ::ActiveRecord::OnDemand::ResultSet
-      def for_reading(options = {})
-        if options.empty?
-          res = query_for_reading(self)
-          if block_given?
-            yield res
-            return
-          end
-          return res
+      module ClassMethods
+        def raw_results
+          query_for_reading self, readonly: true, raw: true
         end
 
-        relation = self
+        # Ripped from the find_in_batches function, but customized to return an ::ActiveRecord::OnDemand::ResultSet
+        def for_reading(options = {})
+          if options.empty?
+            res = query_for_reading self, readonly: true
+            if block_given?
+              yield res
+              return
+            end
+            return res
+          end
 
-        unless arel.orders.blank? && arel.taken.blank?
-          ActiveRecord::Base.logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size")
-        end
+          relation = self
 
-        if (finder_options = options.except(:start, :batch_size)).present?
-          raise "You can't specify an order, it's forced to be #{batch_order_for_reading}" if options[:order].present?
-          raise "You can't specify a limit, it's forced to be the batch_size"  if options[:limit].present?
+          unless arel.orders.blank? && arel.taken.blank?
+            ::ActiveRecord::Base.logger.warn("Scoped order and limit are ignored, it's forced to be batch order and batch size")
+          end
 
-          relation = apply_finder_options(finder_options)
-        end
+          if (finder_options = options.except(:start, :batch_size)).present?
+            raise "You can't specify an order, it's forced to be #{batch_order_for_reading}" if options[:order].present?
+            raise "You can't specify a limit, it's forced to be the batch_size"  if options[:limit].present?
 
-        start = options.delete(:start)
-        batch_size = options.delete(:batch_size) || 1000
+            relation = apply_finder_options(finder_options)
+          end
 
-        relation = relation.reorder(batch_order_for_reading).limit(batch_size)
-        records = query_for_reading(start ? relation.where(table[primary_key].gteq(start)) : relation)
+          start = options.delete(:start)
+          batch_size = options.delete(:batch_size) || 1000
 
-        while records.any?
-          records_size = records.size
-          primary_key_offset = records.last.id
+          relation = relation.reorder(batch_order_for_reading).limit(batch_size)
+          records = query_for_reading(start ? relation.where(table[primary_key].gteq(start)) : relation, readonly: true)
 
-          yield records
+          while records.any?
+            records_size = records.size
+            primary_key_offset = records.last.id
 
-          break if records_size < batch_size
+            yield records
 
-          if primary_key_offset
-            records = query_for_reading relation.where(table[primary_key].gt(primary_key_offset))
-          else
-            raise "Primary key not included in the custom select clause"
+            break if records_size < batch_size
+
+            if primary_key_offset
+              records = query_for_reading relation.where(table[primary_key].gt(primary_key_offset)), readonly: true
+            else
+              raise 'Primary key not included in the custom select clause'
+            end
           end
         end
-      end
 
-      private
+        private
 
-      def query_for_reading(ar)
-        results = ::ActiveRecord::Base.connection.exec_query ar.to_sql
-        ::ActiveRecord::OnDemand::ResultSet.new self.arel.engine, results, readonly: true
-      end
+        def query_for_reading(ar, options = {})
+          ar = ar.scoped unless ar.respond_to?(:to_sql)
+          results = ::ActiveRecord::Base.connection.exec_query ar.to_sql
+          ::ActiveRecord::OnDemand::ResultSet.new ar.arel.engine, results, options
+        end
 
-      def batch_order_for_reading
-        "#{quoted_table_name}.#{quoted_primary_key} ASC"
+        def batch_order_for_reading
+          "#{quoted_table_name}.#{quoted_primary_key} ASC"
+        end
       end
     end
   end
 end
 
-::ActiveRecord::Relation.send(:include, ::ActiveRecord::OnDemand::ForReadingExtension)
+::ActiveRecord::Base.send     :include, ::ActiveRecord::OnDemand::Reading
+::ActiveRecord::Relation.send :include, ::ActiveRecord::OnDemand::Reading::ClassMethods

data/lib/ar-ondemand/for_streaming.rb

@@ -0,0 +1,26 @@
+require 'active_support/concern'
+require 'ar-ondemand/for_reading'
+
+module ActiveRecord
+  module OnDemand
+    module Streaming
+      extend ::ActiveSupport::Concern
+
+      module ClassMethods
+        def for_streaming(options = {})
+          options[:batch_size] ||= 50_000
+          fr = options.delete(:for_reading)
+          s = self.respond_to?(:scoped) ? self.scoped : self
+          ::Enumerator.new do |n|
+            s.send(fr ? :for_reading : :find_in_batches, options) do |b|
+              b.each { |r| n << r }
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+::ActiveRecord::Base.send     :include, ::ActiveRecord::OnDemand::Streaming
+::ActiveRecord::Relation.send :include, ::ActiveRecord::OnDemand::Streaming::ClassMethods

data/lib/ar-ondemand/result.rb

@@ -3,11 +3,14 @@ module ActiveRecord
     class ResultSet
       include ::Enumerable
 
+      CACHED_READONLY_CLASSES ||= {}
+
       def initialize(model, results, options = {})
         @model = model
         @results = results
         @column_types = Hash[@model.columns.map { |x| [x.name, x] }]
         @col_indexes = HashWithIndifferentAccess[@results.columns.each_with_index.map { |x, i| [x,i] }]
+        @raw = options.delete :raw
         @readonly = options.delete :readonly
         @readonly_klass = @readonly ? create_readonly_class : nil
       end
@@ -39,6 +42,7 @@ module ActiveRecord
 
       def result_to_record(row)
         return nil if row.nil?
+        return row if @raw
         if @readonly
           convert_to_struct @readonly_klass, row
         else
@@ -48,7 +52,8 @@ module ActiveRecord
 
       def create_readonly_class
         attrs = @col_indexes.keys.map(&:to_sym)
-        ::Struct.new *attrs
+        return CACHED_READONLY_CLASSES[attrs] if CACHED_READONLY_CLASSES[attrs]
+        CACHED_READONLY_CLASSES[attrs] = ::Struct.new *attrs
       end
 
       def convert_to_hash(rec)

data/lib/ar-ondemand/version.rb

@@ -1,3 +1,3 @@
 module ArOnDemand
-  VERSION = '1.1.3'
+  VERSION = '1.1.4'
 end

data/lib/ar-ondemand.rb

@@ -1,3 +1,4 @@
 require_dependency 'ar-ondemand/on_demand'
 require_dependency 'ar-ondemand/for_reading'
+require_dependency 'ar-ondemand/for_streaming'
 require_dependency 'ar-ondemand/delete_all_by_pk'

data/spec/lib/delete_all_by_pk_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+require 'ar-ondemand'
+
+describe 'DeleteAllByPk' do
+  context 'Testing' do
+    context 'ClassMethod' do
+      before(:each) do
+        create(:audit_record)
+      end
+
+      it 'should delete record' do
+        AuditRecord.delete_all_by_pk.should eq(1)
+      end
+    end
+
+    context 'Relation' do
+      before(:each) do
+        create(:audit_record)
+      end
+
+      it 'should delete record' do
+        AuditRecord.where('id > 0').delete_all_by_pk.should eq(1)
+      end
+    end
+  end
+end

data/spec/lib/for_reading_spec.rb

@@ -19,7 +19,7 @@ describe 'ForReading' do
 
     context 'Iterating' do
       before(:each) do
-        (1..100).each do
+        (1..25).each do
           create(:audit_record)
         end
       end
@@ -29,7 +29,15 @@ describe 'ForReading' do
         AuditRecord.where(customer_id: 1).for_reading.each do |r|
           total += 1
         end
-        total.should eq(100)
+        total.should eq(25)
+      end
+
+      it 'should support iterating' do
+        total = 0
+        AuditRecord.for_reading.each do |r|
+          total += 1
+        end
+        total.should eq(25)
       end
 
       it 'should produce same results as regular iterating' do

data/spec/lib/for_streaming_spec.rb

@@ -0,0 +1,69 @@
+require 'spec_helper'
+require 'ar-ondemand'
+
+describe 'ForStreaming' do
+  context 'Testing' do
+    context 'Iterating' do
+      before(:each) do
+        (1..25).each do
+          create(:audit_record)
+        end
+      end
+
+      context 'AR' do
+        it 'ClassMethod should support streaming' do
+          total = 0
+          AuditRecord.for_streaming.each do |r|
+            #r.is_a?(AuditRecord).should eq(true)
+            expect(r).to be_an_instance_of(AuditRecord)
+            total += 1
+          end
+          total.should eq(25)
+        end
+
+        it 'Relation should support streaming' do
+          total = 0
+          AuditRecord.where(customer_id: 1).for_streaming.each do |r|
+            expect(r).to be_an_instance_of(AuditRecord)
+            total += 1
+          end
+          total.should eq(25)
+        end
+
+        it 'should support select' do
+          total = 0
+          AuditRecord.select([:customer_id]).where(customer_id: 1).for_streaming.each do |r|
+            expect(r).to be_an_instance_of(AuditRecord)
+            expect(r.customer_id).to eq(1)
+            expect { r.model_type_id }.to raise_error(::ActiveModel::MissingAttributeError)
+            total += 1
+          end
+          total.should eq(25)
+        end
+      end
+
+      context 'For Reading' do
+        it 'ClassMethod should support streaming' do
+          total = 0
+          AuditRecord.for_streaming(for_reading: true).each do |r|
+            expect(r).not_to be_an_instance_of(AuditRecord)
+            r.customer_id.should eq(1)
+            total += 1
+          end
+          total.should eq(25)
+        end
+
+        it 'Relation should support streaming' do
+          total = 0
+          AuditRecord.where(customer_id: 1).for_streaming(for_reading: true).each do |r|
+            expect(r).not_to be_an_instance_of(AuditRecord)
+            r.customer_id.should eq(1)
+            total += 1
+          end
+          total.should eq(25)
+        end
+      end
+
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ar-ondemand
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 1.1.4
 platform: ruby
 authors:
 - Steve Frank
@@ -55,6 +55,7 @@ files:
 - lib/ar-ondemand.rb
 - lib/ar-ondemand/delete_all_by_pk.rb
 - lib/ar-ondemand/for_reading.rb
+- lib/ar-ondemand/for_streaming.rb
 - lib/ar-ondemand/on_demand.rb
 - lib/ar-ondemand/record.rb
 - lib/ar-ondemand/result.rb
@@ -64,7 +65,9 @@ files:
 - spec/db/schema.rb
 - spec/db/seeds.rb
 - spec/factories/audit_record.rb
+- spec/lib/delete_all_by_pk_spec.rb
 - spec/lib/for_reading_spec.rb
+- spec/lib/for_streaming_spec.rb
 - spec/spec_helper.rb
 - spec/support/active_record.rb
 homepage: https://github.com/CloudHealth/ar-ondemand