active_record_bulk_insert 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 65eefc8f4eff1019d60778a09edd78d794e8eaec
+  data.tar.gz: 758c122f661de71f7402dab334bdbb176f767915
+SHA512:
+  metadata.gz: 0b86eabb77daa541c1a5e3c0a43c1c954d1e0d1f9c3f54a3f35b4af27b191318c4aa0bb489ea6ab6fdf14244c750300e56f4a1c337c1924b1181e38db787eed2
+  data.tar.gz: 16c35e55835dc81a7e29d1be0a58b5b2c2ce1067a07337d87af2b09f8c4c94e2610ad4a76b67c88b2caf658fe829b7c90853d6f98e28b9cb4f2c7af7891d659b

data/README.md

@@ -0,0 +1,77 @@
+# BULK INSERT
+
+
+### Quick Example
+
+```ruby
+#Gemfile
+gem "active_record_bulk_insert"
+```
+
+```ruby
+users = [{:username => "foo", :firstname => "Foo", :lastname => "Bar", :age => 31},
+         {:username => "j.doe", :firstname => "John", :lastname => "Doe", :age => 57}]
+User.bulk_insert(users)
+User.count # => 2
+```
+
+### Insert ActiveRecord Objects
+
+```ruby
+user1 = User.new({:username => "foo", :firstname => "Foo", :lastname => "Bar", :age => 31})
+user2 = User.new({:username => "j.doe", :firstname => "John", :lastname => "Doe", :age => 57})
+User.bulk_insert([user1, user2])
+User.count # => 2
+```
+
+### Evoke Active Record Validations (after_intialize and validation callbacks)
+
+```ruby
+class User < ActiveRecord
+  validates :username, :presence => true
+end
+users = [{:username => "foo", :firstname => "Foo", :lastname => "Bar", :age => 31},
+         {:username => "j.doe", :firstname => "John", :lastname => "Doe", :age => 57},
+         {:firstname => "John", :lastname => "Doe", :age => 57}]
+User.bulk_insert(users, :validate => true)
+# => [{:firstname => "John", :lastname => "Doe", :age => 57}]
+```
+*The return value is a list of invalid records*
+
+### Provide your own primary keys
+
+```ruby
+users = [{:id => 200, :username => "foo", :firstname => "Foo", :lastname => "Bar", :age => 31},
+         :id => 201, {:username => "j.doe", :firstname => "John", :lastname => "Doe", :age => 57}]
+User.bulk_insert(users, :use_provided_primary_key => true)
+```
+*note this is only available from ActiveRecord 4.0 as id was protected from mass-assignment in ActiveRecord < 4.0*
+
+### Bulk insert in batches
+
+```ruby
+users = 1000000.times.map do |i|
+  {:username => "foo#{i}", :firstname => "Foo#{i}", :lastname => "Bar#{i}", :age => (30..70).to_a.sample}
+end
+User.bulk_insert_in_batches(users, :batch_size => 10000)
+User.count # => 1000000
+```
+
+### Benchmark
+DB: PostgreSQL 9.1.9  
+OS: Debian GNU/Linux 7 (wheezy)  
+Processor: Intel(R) Core(TM) i7-4960HQ CPU @ 2.60GHz  
+Memory: 6GB  
+Number of Records: 10,000  
+
+```
+                                user     system      total        real
+Create with Active Record   1.970000   6.810000   8.780000 ( 14.822348)
+AR with validations         0.280000   0.000000   0.280000 (  0.299018)
+AR without validations      0.190000   0.000000   0.190000 (  0.207807)
+Hash without validations    0.080000   0.000000   0.080000 (  0.108874)
+Hash with validations       0.690000   0.000000   0.690000 (  0.669952)
+```
+
+License
+bulk_insert is released under [MIT license.](http://opensource.org/licenses/MIT)

data/lib/base.rb

@@ -0,0 +1,53 @@
+ActiveRecord::Base.class_eval do
+  def self.bulk_insert_in_batches(attrs, options = {})
+    batch_size = options.fetch(:batch_size, 1000)
+
+    attrs.each_slice(batch_size).map do |sliced_attrs|
+      bulk_insert(sliced_attrs, options)
+    end.flatten.compact
+  end
+
+  def self.bulk_insert(attrs, options = {})
+    use_provided_primary_key = options.fetch(:use_provided_primary_key, false)
+    attributes = _resolve_record(attrs.first, use_provided_primary_key).keys.join(", ")
+
+    if options.fetch(:validate, false)
+      attrs, invalid = attrs.partition { |record| _validate(record) }
+    end
+
+    values_sql = attrs.map do |record|
+      "(#{_resolve_record(record, use_provided_primary_key).values.map { |r| sanitize(r) }.join(', ')})"
+    end.join(",")
+
+    sql = <<-SQL
+      INSERT INTO #{quoted_table_name}
+        (#{attributes})
+      VALUES
+        #{values_sql}
+    SQL
+    connection.execute(sql) unless attrs.empty?
+    invalid
+  end
+
+  def self._resolve_record(record, use_provided_primary_key)
+    if record.is_a?(Hash) && use_provided_primary_key
+      record.except(primary_key).except(primary_key.to_sym)
+    elsif record.is_a?(Hash)
+      record
+    elsif record.is_a?(ActiveRecord::Base) && use_provided_primary_key
+      record.attributes
+    elsif record.is_a?(ActiveRecord::Base)
+      record.attributes.except(primary_key).except(primary_key.to_sym)
+    end
+  end
+
+  def self._validate(record)
+    if record.is_a?(Hash)
+      new(record).valid?
+    elsif record.is_a?(ActiveRecord::Base)
+      record.valid?
+    else
+      false
+    end
+  end
+end

data/spec/sample_record_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+describe SampleRecord do
+ describe "bulk_insert" do
+   it "inserts records into the DB" do
+     ActiveRecord::ConnectionAdapters::SQLite3Adapter.any_instance.should_receive(:execute).with do |params|
+       params.should include("INSERT INTO \"sample_records\"")
+       params.should include("Foo")
+       params.should include("30")
+     end.once
+     SampleRecord.bulk_insert([{:name => "Foo", :age => 30}])
+   end
+
+   it "inserts records into the DB and increases count of records" do
+     records = 5.times.map { |i| SampleRecord.new(:age => i + (30..50).to_a.sample, :name => "Foo#{i}").attributes }
+     expect {SampleRecord.bulk_insert(records)}.to change{SampleRecord.count}.by(records.size)
+   end
+
+   it "inserts multiple records into the DB in a single insert statement" do
+     records = 10.times.map { |i| {:age => 4, :name => "Foo#{i}"} }
+
+     ActiveRecord::ConnectionAdapters::SQLite3Adapter.any_instance.should_receive(:execute).with do |params|
+       matchdata = params.match(/insert into "sample_records"/i)
+       matchdata.to_a.count.should == 1
+       records.each do |record|
+         params.should include(record[:age].to_s)
+         params.should include(record[:name])
+       end
+     end.once
+
+     SampleRecord.bulk_insert(records)
+   end
+
+   it "relies on the DB to provide primary_key if :use_provided_primary_key is false or nil" do
+     records = 10.times.map { |i| SampleRecord.new(:id => 10000 + i, :age => 4, :name => "Foo#{i}") }
+
+     ActiveRecord::ConnectionAdapters::SQLite3Adapter.any_instance.should_receive(:execute).with do |params|
+       records.each do |record|
+         params.should_not include(record.id.to_s)
+       end
+     end
+
+     SampleRecord.bulk_insert(records)
+   end
+
+   it "uses provided primary_key if :use_provided_primary_key is true" do
+     records = 10.times.map { |i| SampleRecord.new(:id => 10000 + i, :age => 4, :name => "Foo#{i}") }
+
+     SampleRecord.bulk_insert(records, :use_provided_primary_key => true)
+     records.each do |record|
+       SampleRecord.exists?(:id => record.id).should be_true
+     end
+   end
+
+   it "support insertion of ActiveRecord objects" do
+     records = 10.times.map { |i| SampleRecord.new(:age => 4, :name => "Foo#{i}") }
+
+     ActiveRecord::ConnectionAdapters::SQLite3Adapter.any_instance.should_receive(:execute).with do |params|
+       matchdata = params.match(/insert into "sample_records"/i)
+       matchdata.to_a.count.should == 1
+       records.each do |record|
+         params.should include(record.age.to_s)
+         params.should include(record.name)
+       end
+     end.once
+
+     SampleRecord.bulk_insert(records)
+   end
+
+   context "validations" do
+     it "should not persist invalid records if ':validate => true' is specified" do
+       SampleRecord.send(:validates, :name, :presence => true)
+       expect {SampleRecord.bulk_insert([:age => 30], :validate => true)}.to_not change{SampleRecord.count}
+     end
+   end
+ end
+
+ describe "bulk_insert_in_batches" do
+   it "allows you to specify a batch_size" do
+     records = 10.times.map { |i| SampleRecord.new(:age => 4, :name => "Foo#{i}").attributes }
+
+     ActiveRecord::ConnectionAdapters::SQLite3Adapter.any_instance.should_receive(:execute).with do |params|
+       params.should include("INSERT INTO \"sample_records\"")
+     end.exactly(5).times
+
+     SampleRecord.bulk_insert_in_batches(records, :batch_size => 2)
+   end
+ end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,27 @@
+require "active_record"
+require 'base'
+require "database_cleaner"
+require "support/sample_record"
+
+ActiveRecord::Base.establish_connection(:adapter => "sqlite3", :database => ":memory:")
+ActiveRecord::Migration.verbose = false
+ActiveRecord::Migrator.migrate("spec/support/migrations")
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.order = "random"
+
+  config.before(:suite) do
+    DatabaseCleaner.strategy = :transaction
+    DatabaseCleaner.clean_with(:truncation)
+  end
+
+  config.before(:each) do
+    DatabaseCleaner.start
+  end
+
+  config.after(:each) do
+    DatabaseCleaner.clean
+  end
+end
+

data/spec/support/migrations/1_sample_record_migration.rb

@@ -0,0 +1,9 @@
+class SampleRecordMigration < ActiveRecord::Migration
+  def change
+    create_table :sample_records do |t|
+      t.text "name"
+      t.integer "age"
+    end
+  end
+end
+

data/spec/support/sample_record.rb

@@ -0,0 +1,2 @@
+class SampleRecord < ActiveRecord::Base
+end

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification
+name: active_record_bulk_insert
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Abejide Ayodele
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.13.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.13.0
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.3.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.3.7
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.17.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.17.1
+description: Exposes a bulk insert API to AR subclasses
+email:
+- abejideayodele@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/base.rb
+- spec/sample_record_spec.rb
+- spec/spec_helper.rb
+- spec/support/migrations/1_sample_record_migration.rb
+- spec/support/sample_record.rb
+- README.md
+homepage: https://github.com/bjhaid/active_record_bulk_insert
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- - lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: bulk insert records into the DB
+test_files:
+- spec/sample_record_spec.rb
+- spec/spec_helper.rb
+- spec/support/migrations/1_sample_record_migration.rb
+- spec/support/sample_record.rb