MikeSofaer-sax-mapper 0.0.1 → 0.0.2

data/README

@@ -8,6 +8,8 @@ You can also set a column as a remote primary key, and it will overwrite rather
 than add records where that key is a duplicate (you need to separately set
 the column to be have a unique index, since it uses ON DUPLICATE KEY UPDATE)
 
+You can specify a batch size to save in, and whether to wrap the save in a transaction.
+
 Finally, you can mark fields as required, and it will raise an expection if the
 XML is missing that field.
 
@@ -33,9 +35,9 @@ end
 Person.parse_multiple(xml) will return an array of Person objects, found inside
 <person></person> tags.  You can save them to the DB with Person.save(array)
 
-Multiple values with the same sourced_id will replace each other in the DB.
+Person.save(array, :batch_size => 3000, :transaction => true) will try to save them in batches of 3000, and roll everything back if any save fails.
 
-Query batching is not built in, you have to slice the array.
+Multiple values with the same sourced_id will replace each other in the DB.
 
 gem install MikeSofaer-sax-mapper
 

data/lib/sax-mapper.rb

@@ -1,5 +1,6 @@
 require 'sax-machine'
 require 'dm-core'
+require 'enumerator'
 
 module SaxMapper
   class MissingElementError < Exception; end
@@ -80,8 +81,19 @@ module SaxMapper
       " ON DUPLICATE KEY UPDATE " + (column_names - [:created_at, @key_column]).map {|c| c.to_s + "=VALUES(" + c.to_s + ")"}.join(', ')
     end
 
-    def save(rows)
-      connection.execute sql(rows), *bind_values(rows)
+    def save(rows, options = {})
+      connection.execute "BEGIN" if options[:transaction]
+      if options[:batch_size]
+        rows.each_slice(options[:batch_size]) do |batch|
+          connection.execute sql(batch), *bind_values(batch)
+        end
+      else
+        connection.execute sql(rows), *bind_values(rows)
+      end
+      connection.execute "COMMIT" if options[:transaction]
+    rescue Exception => e
+      connection.execute "ROLLBACK" if options[:transaction]
+      raise e
     end
   end
 

data/spec/sax-mapper/sax-mapper_spec.rb

@@ -70,9 +70,30 @@ describe "SaxMapper" do
           @klass.datamapper_class.all[0].title.should == "Hello, Everyone!"
           @klass.datamapper_class.all[1].title.should == "Someone's Cat"
         end
+        it "should support chunking into multiple queries" do
+          documents = @klass.parse_multiple(@xml)
+          @adapter.should_receive(:execute).exactly(3).times
+          @klass.save documents, :batch_size => 1
+        end
+        it "should work when chunking" do
+          documents = @klass.parse_multiple(@xml)
+          @klass.save documents
+          @klass.datamapper_class.all[0].title.should == "Hello, Everyone!"
+          @klass.datamapper_class.all[1].title.should == "Someone's Cat"
+        end
+        it "should do the chunks in a transaction if asked" do
+          t = DateTime.now.to_s
+          @adapter.execute "create unique index key_column on documents(written_on)"
+          xml= "<xml><document><title>Hello, Everyone!</title><written_on>#{t}</written_on></document><document><title>Someone's Cat</title><written_on>#{t}</written_on></document></xml>"
+          documents = @klass.parse_multiple xml
+          go = lambda{@klass.save documents, :batch_size => 1, :transaction => true}
+          go.should raise_error
+          @klass.datamapper_class.all.size.should == 0
+        end
       end
       describe "replication" do
         it "should update fields on rows with a repeated primary key" do
+          DataMapper.auto_migrate!
           @klass.key_column :written_on
           @adapter.execute "create unique index key_column on documents(written_on)"
           t = DateTime.now.to_s

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: MikeSofaer-sax-mapper
 version: !ruby/object:Gem::Version 
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors: 
 - Michael Sofaer