activerecord_save_many 0.2.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Trampoline Systems Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,18 @@
+= activerecord_save_many
+
+adds a save_many method to ActiveRecord classes. save_many generates reasonable
+SQL for inserting or updating many ActiveRecord instances in one go
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 Trampoline Systems Ltd. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,46 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "activerecord_save_many"
+    gem.summary = %Q{efficient bulk inserts and updates for ActiveRecord}
+    gem.description = %Q{adds save_many method to ActiveRecord classes for efficient bulk insert and update operations}
+    gem.email = "craig@trampolinesystems.com"
+    gem.homepage = "http://github.com/trampoline/activerecord_save_many"
+    gem.authors = ["mccraigmccraig"]
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_development_dependency "yard", ">= 0"
+    gem.add_development_dependency "rr", ">= 0.10.5"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+begin
+  require 'yard'
+  YARD::Rake::YardocTask.new
+rescue LoadError
+  task :yardoc do
+    abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
+  end
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/lib/activerecord_save_many.rb

@@ -0,0 +1,108 @@
+require 'set'
+
+module ActiveRecord
+  module SaveMany
+    MAX_QUERY_SIZE = 1024 * 1024
+    OPTIONS_KEYS = [:columns, :max_rows, :async, :ignore, :update, :updates].to_set
+
+    class << self
+      attr_accessor :default_max_rows
+    end
+    self.default_max_rows = 50000
+
+    module Functions
+      def rails_env
+        RAILS_ENV if defined? RAILS_ENV
+      end
+      module_function :rails_env
+
+      def disable_async?
+        # for predictable tests we disable delayed insert during testing
+        rails_env()=="test"
+      end
+      module_function :disable_async?
+
+      def check_options(options)
+        unknown_keys = options.keys.to_set - OPTIONS_KEYS.to_set
+        raise "unknown options: #{unknown_keys.to_a.join(", ")}" if !unknown_keys.empty?
+      end
+      module_function :check_options
+
+      # slice an array into smaller arrays with maximum size max_size
+      def slice_array(max_length, arr)
+        slices = []
+        (0..arr.length-1).step( max_length ){ |i| slices << arr.slice(i,max_length) }
+        slices
+      end
+      module_function :slice_array
+
+      def add_columns(klass, values, options)
+        columns = options[:columns] || klass.columns.map(&:name)
+
+        # add a :type column automatically for STI, if not already present
+        if klass.superclass!=ActiveRecord::Base && !columns.include?(:type)
+          columns = [:type, *columns]
+          values = values.map{|vals| [klass.to_s, *vals]}
+        end
+
+        [columns, values]
+      end
+      module_function :add_columns
+    end
+
+    module ClassMethods
+      def save_many_max_rows=(max_rows)
+        @save_many_max_rows=max_rows
+      end
+
+      def save_many_max_rows
+        @save_many_max_rows || ActiveRecord::SaveMany::default_max_rows
+      end
+
+      def save_many(values, options={})
+        Functions::check_options(options)
+        return if values.nil? || values.empty?
+
+        columns, values = Functions::add_columns(self, values, options)
+
+        # if more than max_rows, execute multiple sql statements
+        max_rows = options[:max_rows] || save_many_max_rows
+        batches = Functions::slice_array(max_rows, values)
+
+        column_list = columns.join(', ')
+        do_updates = options[:update] || options[:updates]
+        updates = options[:updates] || {}
+
+        batches.each do |batch|
+          batch = batch.map do |obj|
+            if obj.is_a? ActiveRecord::Base
+              obj.send( :callback, :before_save )
+              if obj.id
+                obj.send( :callback, :before_update)
+              else
+                obj.send( :callback, :before_create )
+              end
+              raise "#{obj.errors.full_messages.join(', ')}" if !obj.valid?
+            end
+            columns.map{|col| obj[col]}
+          end
+
+          insert_stmt = options[:async] && !disable_async? ? "insert delayed" : "insert"
+          ignore_opt = options[:ignore] ? "ignore" : ""
+          
+          sql = "#{insert_stmt} #{ignore_opt} into #{table_name} (#{column_list}) values " + 
+            batch.map{|vals| "(" + vals.map{|v| quote_value(v)}.join(", ") +")"}.join(", ") +
+            (" on duplicate key update " + columns.map{|c| updates[c] || " #{c} = values(#{c}) "}.join(", ") if do_updates).to_s
+
+          connection.execute_raw sql
+        end
+      end
+    end
+  end
+
+  class Base
+    class << self
+      include ActiveRecord::SaveMany::ClassMethods
+    end
+  end
+end

data/spec/activerecord_save_many_spec.rb

@@ -0,0 +1,104 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+module ActiveRecord
+  describe SaveMany do
+    def new_anon_class(parent, &proc)
+      klass = Class.new(parent)  
+      klass.class_eval(&proc) if proc
+      klass
+    end
+
+    def new_named_class(parent, name="", &proc)
+      new_anon_class(parent) do
+        mc=self.instance_eval{ class << self ; self ; end }
+        mc.send(:define_method, :to_s){name}
+        self.class_eval(&proc) if proc
+      end
+    end
+
+    # create a new anonymous ActiveRecord::Base descendant class
+    def new_ar_class(name="", &proc)
+      new_named_class(ActiveRecord::Base, name, &proc)
+    end
+
+    describe SaveMany::Functions do
+      describe "disable_async?" do
+        it "should disable async inserts when testing" do
+          mock(SaveMany::Functions).rails_env{"test"}
+          SaveMany::Functions::disable_async?.should == true
+        end
+
+        it "should permit async inserts when not testing" do
+          mock(SaveMany::Functions).rails_env{"production"}
+          SaveMany::Functions::disable_async?.should == false
+        end
+      end
+
+      describe "check_options" do
+        it "should raise if given unknown options" do
+          lambda do
+            SaveMany::Functions::check_options(:foo=>100)
+          end.should raise_error(RuntimeError)
+        end
+      end
+
+      describe "slice_array" do
+        it "should slice arrays without losing bits" do
+          SaveMany::Functions::slice_array(2,[]).should ==([])
+          SaveMany::Functions::slice_array(2,[1]).should ==([[1]])
+          SaveMany::Functions::slice_array(2,[1,2]).should ==([[1,2]])
+          SaveMany::Functions::slice_array(2,[1,2,3]).should ==([[1,2],[3]])
+          SaveMany::Functions::slice_array(2,[1,2,3,4]).should ==([[1,2],[3,4]])
+        end
+      end
+
+      describe "add_columns" do
+        it "should add a type column to an indirect inheritor of ActiveRecord::Base" do
+          klass = new_named_class(new_ar_class("Foo"), "Bar")
+          columns, values = SaveMany::Functions::add_columns(klass, [["foo"]], :columns=>[:foo])
+          columns.should == [:type, :foo]
+          values.should == [["Bar", "foo"]]
+        end
+
+        it "should not add a type column if already present" do
+          klass = new_named_class(new_ar_class("Foo"), "Bar")
+          columns, values = SaveMany::Functions::add_columns(klass, [["foo", "Baz"]], :columns=>[:foo, :type])
+          columns.should == [:foo, :type]
+          values.should == [["foo", "Baz"]]
+        end
+
+        it "should not add a type column to a direct inheritor of ActiveRecord::Base" do
+          klass = new_ar_class("Foo")
+          columns, values = SaveMany::Functions::add_columns(klass, [["foo"]], :columns=>[:foo])
+          columns.should == [:foo]
+          values.should == [["foo"]]
+        end
+      end
+    end
+
+    it "creates a save_many method on an ActiveRecord class" do
+      klass = new_ar_class()
+      klass.respond_to?(:save_many).should be(true)
+    end
+
+    it "should have per-class configurable save_many_max_rows" do
+      k1 = new_ar_class()
+      k2 = new_ar_class()
+      k1.save_many_max_rows=1000
+      k2.save_many_max_rows=2000
+
+      k1.save_many_max_rows.should == 1000
+      k2.save_many_max_rows.should == 2000
+    end
+
+    it "should have a global configurable default_max_rows" do
+      k1 = new_ar_class()
+      k2 = new_ar_class()
+      ActiveRecord::SaveMany::default_max_rows = 100
+      k1.save_many_max_rows.should == 100
+      k2.save_many_max_rows.should == 100
+    end
+
+    
+  end
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'activerecord_save_many'
+require 'spec'
+require 'spec/autorun'
+
+Spec::Runner.configure do |config|
+  config.mock_with :rr
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification 
+name: activerecord_save_many
+version: !ruby/object:Gem::Version 
+  version: 0.2.0
+platform: ruby
+authors: 
+- mccraigmccraig
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-02-17 00:00:00 +00:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.9
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: yard
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: rr
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.10.5
+    version: 
+description: adds save_many method to ActiveRecord classes for efficient bulk insert and update operations
+email: craig@trampolinesystems.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/activerecord_save_many.rb
+- spec/activerecord_save_many_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/trampoline/activerecord_save_many
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: efficient bulk inserts and updates for ActiveRecord
+test_files: 
+- spec/activerecord_save_many_spec.rb
+- spec/spec_helper.rb