activerecord_bulkoperation 0.0.2

data/lib/activerecord_bulkoperation/util/flush_dirty_objects.rb

@@ -0,0 +1,126 @@
+# It's used to store the modified/deleted entity objects at transaction level. 
+# Before the transaction will be closed the stored enities will be send to the database.
+#
+# Author:: Andre Kullmann
+#
+module ActiveRecord
+  module Bulkoperation
+    module Util
+      class FlushDirtyObjects < Util::TransactionObject
+        class DirtyObjects
+          attr_reader :insert_or_update
+          attr_reader :insert_on_missing
+
+          def initialize
+            @insert_or_update  = Set.new
+            @insert_on_missing = {}
+          end
+        end
+
+        def initialize
+          @scheduled_merges    = ActiveRecord::Bulkoperation::Util::EntityHash.new
+          @scheduled_deletions = ActiveRecord::Bulkoperation::Util::EntityHash.new
+        end
+
+        def before_commit
+          flush
+        end
+
+        def before_create_savepoint
+          flush
+        end
+
+        def after_rollback_to_savepoint
+          close
+        end
+
+        def after_rollback
+          close
+        end
+
+        def close
+
+          @scheduled_merges.values.each do |objects|
+            if objects
+              objects.insert_or_update.each { |record| record.on_closed_scheduled_operation }
+              objects.insert_on_missing.values.each { |array| array.each{ |record| record.on_closed_scheduled_operation } }
+            end
+          end
+          @scheduled_merges.clear
+
+          @scheduled_deletions.values.each { |array| array and array.each { |record| record.on_closed_scheduled_operation } }
+          @scheduled_deletions.clear
+        end
+
+        def add_insert_on_missing(keys,object)
+          fail 'nil object' unless object
+          ( ( @scheduled_merges[ object.class] ||= DirtyObjects.new).insert_on_missing[keys] ||= Set.new ) << object
+        end
+
+        def add_merge(object)
+          fail 'nil object' unless object
+          ( @scheduled_merges[ object.class] ||= DirtyObjects.new).insert_or_update << object
+        end
+
+        def add_delete(object)
+          fail 'nil object' unless object
+          ( @scheduled_deletions[ object.class] ||= []) << object
+        end
+
+        def flush(args = {})
+          fail 'nil object' unless args
+
+          affected = 0
+
+          @scheduled_deletions.each_pair_in_detail_hierarchy do |k, v|
+            affected += k.delete_group(v, args)
+            @scheduled_deletions[k] = nil
+            v.each { |record| record.on_closed_scheduled_operation }
+          end
+
+          @scheduled_merges.each_pair_in_master_hierarchy do |k, v|
+
+            affected += k.merge_group( v.insert_or_update, args )
+
+            v.insert_on_missing.each_pair do |keys,records|
+
+              affected += k.insert_on_missing_group( keys, records, args )
+            end
+
+            @scheduled_merges[k] = nil
+            v.insert_or_update.each  { |record| record.on_closed_scheduled_operation }
+            v.insert_on_missing.values.each { |array| array.each{ |record| record.on_closed_scheduled_operation } }
+          end
+
+          affected
+        end
+
+        def flush_record_class(record_class, args = {})
+          fail 'nil object' unless record_class
+          fail 'nil object' unless args
+
+          if list = @scheduled_deletions[ record_class]
+            record_class.delete_group(list, args)
+            @scheduled_deletions[ record_class] = nil
+            list.each { |record| record.on_closed_scheduled_operation }
+          end
+
+          if objects = @scheduled_merges[ record_class]
+
+            record_class.merge_group( objects.insert_or_update, args )
+
+            objects.insert_on_missing.each_pair do |keys,records|
+
+              record_class.insert_on_missing_group( keys, records, args )
+            end
+
+            @scheduled_merges[record_class] = nil
+            objects.insert_or_update.each  { |record| record.on_closed_scheduled_operation }
+            objects.insert_on_missing.values.each { |array| array.each{ |record| record.on_closed_scheduled_operation } }
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/activerecord_bulkoperation/util/sequence_cache.rb

@@ -0,0 +1,52 @@
+# Will fetch the next N values from a sequence and store it in a queue. 
+# Now the values will can be fetched from the queue without any database roundtrip. 
+# If the queue is empty, the next N values will be fetched from database and stored in the queue.
+#
+# It's using the DUAL table so it should only work with oracle.
+# 
+# Author:: Andre Kullmann
+#
+module ActiveRecord
+  module Bulkoperation
+    module Util
+      class SequenceCache        
+
+        #
+        # seq - the sequence name to use with the SequenceCache object
+        # refetch - how many values should be fetch from the database with each roundtrip 
+        def initialize(seq, prefetch = 10)          
+          @seq, @prefetch = seq, prefetch
+          @queue = Queue.new
+        end
+
+        #
+        # return - the next value from the sequence
+        def next_value          
+          while ( value = next_value_from_queue ).nil?
+            fill_queue
+          end
+          value 
+        end
+
+        private
+
+        def next_value_from_queue
+	        @queue.pop(true) 
+        rescue ThreadError => e
+          nil
+        end
+
+        def fill_queue
+          fetch.each do |f|
+            @queue << f
+          end
+        end
+
+        def fetch
+          st = ActiveRecord::Base.connection.exec_query("SELECT #{@seq}.nextval id FROM dual connect by level <= :a", "SQL", [[nil,@prefetch]])
+          st.map {|r| r['id']}
+        end
+      end
+    end
+  end
+end

data/lib/activerecord_bulkoperation/util/transaction_object.rb

@@ -0,0 +1,36 @@
+# A TransactionObject belongs to a connection. It's like a singleton for each transaction.
+#
+# Author:: Andre Kullmann
+#
+module ActiveRecord
+  module Bulkoperation
+    module Util    
+      class TransactionObject
+        def self.get
+          result = ActiveRecord::Base.connection.connection_listeners.select { |l| l.class == self }.first
+          unless result
+            result = new
+            ActiveRecord::Base.connection.connection_listeners << result
+          end
+          result
+        end
+
+        def after_commit
+          close
+          ActiveRecord::Base.connection.connection_listeners.delete(self)
+        end
+
+        def after_rollback
+          close
+          ActiveRecord::Base.connection.connection_listeners.delete(self)
+        end
+
+        def after_rollback_to_savepoint
+        end
+
+        def close
+        end
+      end
+    end
+  end
+end

data/lib/activerecord_bulkoperation/version.rb

@@ -0,0 +1,5 @@
+module ActiveRecord
+  module Bulkoperation
+    VERSION = "0.0.2"
+  end
+end

data/test/active_record_connection_test.rb

@@ -0,0 +1,41 @@
+require File.expand_path(File.dirname(__FILE__) + '/test_helper')
+
+class ActiveRecordConnectionTest < ActiveSupport::TestCase
+  def test_method_definition
+    assert ActiveRecord::Base.connection.respond_to? :connection_listeners 
+    assert ActiveRecord::Base.connection.respond_to? :commit_db_transaction
+    assert ActiveRecord::Base.connection.respond_to? :rollback_db_transaction
+    assert ActiveRecord::Base.connection.respond_to? :rollback_to_savepoint
+    assert ActiveRecord::Base.connection.respond_to? :create_savepoint
+  
+    assert ActiveRecord::Base.connection.respond_to? :commit_db_transaction_without_callback
+    assert ActiveRecord::Base.connection.respond_to? :rollback_db_transaction_without_callback
+    assert ActiveRecord::Base.connection.respond_to? :rollback_to_savepoint_without_callback
+    assert ActiveRecord::Base.connection.respond_to? :create_savepoint_without_callback
+  end
+
+  class MyTestClass
+    def before_commit
+      puts "get calllsslsl"
+    end
+
+    def after_rollback
+      puts "get calllsslsl"
+    end
+  end
+
+  def test_method_calls_definition    
+    test_class = MyTestClass.new
+    test_class.expects(:before_commit)
+    test_class.expects(:after_commit)
+
+    test_class.expects(:after_rollback)
+
+    ActiveRecord::Base.connection.connection_listeners << test_class
+    
+    ActiveRecord::Base.connection.rollback_db_transaction
+
+    ActiveRecord::Base.connection.commit_db_transaction
+    ActiveRecord::Base.connection.connection_listeners.clear
+  end
+end

data/test/adapters/oracle_enhanced.rb

@@ -0,0 +1 @@
+ENV["DB_ADAPTER"] = "oracle_enhanced"

data/test/bulkoperation_test.rb

@@ -0,0 +1,176 @@
+require File.expand_path(File.dirname(__FILE__) + '/test_helper')
+
+class BulkoperationTest < ActiveSupport::TestCase
+
+  def teardown
+    TestTable.delete_all
+  end
+
+  def test_bulk_operation_methods    
+    assert ActiveRecord::Base.respond_to? :flush_scheduled_operations
+    test_obj = TestTable.new
+    assert test_obj.respond_to? :schedule_merge
+  end
+
+  def test_bulk_operation_workflow
+    test_obj = TestTable.new
+    test_obj.author_name = 'test-workflow'
+    test_obj.schedule_merge
+
+    TestTable.flush_scheduled_operations
+    record = TestTable.first
+    assert_equal('test-workflow',record.author_name)    
+  end
+
+  def test_multi_thread
+    thread_count = 5
+    inner_count = 10
+    threads = []
+    thread_count.times do |t|
+      t = Thread.new do
+        inner_count.times do |i|
+          test_obj = TestTable.new
+          test_obj.author_name = "t1-#{1}"
+          test_obj.schedule_merge  
+        end
+        TestTable.flush_scheduled_operations
+      end
+      threads << t
+    end
+    threads.each do |t|
+      t.join
+    end
+    
+    assert_equal (thread_count * inner_count) , TestTable.all.count      
+  end
+
+  def test_multi_update
+    thread_count = 5
+    inner_count = 10
+    threads = []
+    thread_count.times do |t|
+      t = Thread.new do
+        inner_count.times do |i|
+          test_obj = TestTable.new
+          test_obj.author_name = "t1-#{1}"
+          test_obj.schedule_merge  
+        end
+        TestTable.flush_scheduled_operations
+      end
+      threads << t
+    end
+    threads.each do |t|
+      t.join
+    end
+    
+    assert_equal (thread_count * inner_count) , TestTable.all.count
+    TestTable.all.each do |t|
+      t.author_name = "tr-fff"
+      t.schedule_merge
+    end
+    TestTable.flush_scheduled_operations      
+  end
+
+  def test_update_fk_relation
+    #some problems during database creation and recreation
+    return
+    group = Group.new
+    group.schedule_merge    
+    test_obj = TestTable.new
+    test_obj.author_name = 'test-1'
+    test_obj.group_id = group.id
+    test_obj.schedule_merge  
+
+    ActiveRecord::Bulkoperation::Util::FlushDirtyObjects.get.flush
+    
+    TestTable.flush_scheduled_operations
+
+    
+    first = TestTable.first
+    assert_equal('test-1',test_obj.author_name)
+    
+    first.author_name = 'test-2'
+    first.schedule_merge    
+    TestTable.flush_scheduled_operations
+
+    assert_equal('test-2',first.author_name)    
+  end
+
+  def test_connection_listener_get_called 
+    test_obj = TestTable.new
+    test_obj.author_name = 'test-1'    
+    test_obj.schedule_merge  
+
+    assert_equal(0,TestTable.count)
+    ActiveRecord::Base.connection.commit_db_transaction
+    assert_equal(1,TestTable.count)
+    first = TestTable.first
+    assert_equal('test-1',test_obj.author_name)
+    TestTable.delete_all    
+  end
+
+  def test_schedule_merge_relation
+    group = Group.new
+    group.schedule_merge    
+    test_obj = TestTable.new
+    test_obj.author_name = 'test-1'
+    group.test_tables.schedule_merge(test_obj)
+    
+    ActiveRecord::Bulkoperation::Util::FlushDirtyObjects.get.flush  
+  end
+
+  def test_schedule_merge_has_and_belongs_to_many_relation
+    part = Part.new
+    part.schedule_merge
+    assembly = Assembly.new
+    part.assemblies.schedule_merge(assembly)
+    ActiveRecord::Bulkoperation::Util::FlushDirtyObjects.get.flush
+    db_part = Part.first    
+    db_assembly = Assembly.first
+
+    assert_equal(1,db_part.assemblies.count)
+    assert_equal(db_assembly[:id],db_part.assemblies.first[:id])
+  
+    assert_equal(1,db_assembly.parts.count)
+    assert_equal(db_part[:id],db_assembly.parts.first[:id])    
+  end
+
+  def test_schedule_merge_has_and_belongs_to_many_relation_custom_table_and_columns
+    
+    course = Course.new
+    course.course_id = 10
+    course.schedule_merge
+    student = Student.new
+    student.student_id = 12
+    course.students.schedule_merge(student)
+    ActiveRecord::Bulkoperation::Util::FlushDirtyObjects.get.flush
+    return 
+    db_part = Course.first    
+    db_assembly = Assembly.first
+
+    assert_equal(1,db_part.assemblies.count)
+    assert_equal(db_assembly[:id],db_part.assemblies.first[:id])
+  
+    assert_equal(1,db_assembly.parts.count)
+    assert_equal(db_part[:id],db_assembly.parts.first[:id])    
+  end
+
+  def test_schedule_merge_has_and_belongs_to_many_relation_self_join    
+    product = Product.new
+    product2 = Product.new
+    product.schedule_merge
+    product.related_products.schedule_merge(product2)
+    ActiveRecord::Bulkoperation::Util::FlushDirtyObjects.get.flush
+    return 
+    db_part = Course.first    
+    db_assembly = Assembly.first
+
+    assert_equal(1,db_part.assemblies.count)
+    assert_equal(db_assembly[:id],db_part.assemblies.first[:id])
+  
+    assert_equal(1,db_assembly.parts.count)
+    assert_equal(db_part[:id],db_assembly.parts.first[:id])    
+  end
+
+end
+

data/test/database.yml

@@ -0,0 +1,8 @@
+oracle_enhanced:
+  adapter: oracle_enhanced
+  driver: oracle.jdbc.driver.OracleDriver
+  url: jdbc:oracle:thin:@localhost:1521:ORCL
+  username: test
+  password: test
+  database: ORCL
+  pool: 200