mass_insert 0.1.3 → 0.2.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    Zjc0YmM2NTg0MTRmYjg2NWQyNTcxMWQzZjdhY2VmMzVjOGE1ODFlNg==
-  data.tar.gz: !binary |-
-    ZTE4MGExN2IwY2NmOTBkYTUwNmVmYzNjZDZkM2IxZTNhMmY1MmYyNw==
+SHA1:
+  metadata.gz: 048a8940b6950ce71e156538ee9e3e49fc4093e7
+  data.tar.gz: c06223f8ded6fb2f4596fdb1514a57c02c90038e
 SHA512:
-  metadata.gz: !binary |-
-    NzM2ZjNiODg1ODFkYjk0ZmJlM2Y1NGQ2MmI3Zjg1N2ZkYzU2OTBjMDM5MTUx
-    OTQxZDI4MTEyZGI2MTFjZWI5NzJiMWE2YzQxMTJiZTliZWYyMzIyOTEzNTkx
-    ZWE0MTZkYjIxODE2ZDMyYTJkMDE0Mzg4NTRhZDQ2MmVlNDNiZDM=
-  data.tar.gz: !binary |-
-    M2Q1MjFlYmVhNzc2ZDRlNDY4MTM0MTk3NjA0ODg4NDNkYmMwZDU4YjJkNWZm
-    N2IxZTZhMTI3MGFiNjI3Y2RjZGM1ZWUyYzNlODlkOTEzMDliMzM3ZjA2NzAz
-    MTA0ZmVjMmExNjNmNWYwNDE3MTRiNmJkYzMxYTEzMjI1OGMzYmU=
+  metadata.gz: 8016065eb3fc0076411860b949f4e0499a0754f7b3c4e2469573dc3094eb3f40a81fe2d50f60d947318c9db2504d648ef2ad17750914c631dc008d4bf37c151b
+  data.tar.gz: fe0d19f977031e25ae993d9ccc8bb2534d442ca0cea307c63afc4bc3630cd4c4d036a41a7037e69b06fdcefe151e84adb7ac4b7c0399cb98d2e1b3c5d28a9f98

data/.gitignore

@@ -8,12 +8,14 @@ InstalledFiles
 _yardoc
 coverage
 doc/
+log/
 lib/bundler/man
 pkg
 rdoc
 spec/reports
 test/tmp
 test/version_tmp
+test/database.yml
 tmp
 .rvmrc
 .rspec

data/.travis.yml

@@ -1,14 +1,16 @@
 language: ruby
 rvm:
-  - "1.9.2"
-  - "1.9.3"
-  - "2.0.0"
+  - '1.9.3'
+  - '2.0.0'
+  - '2.1.0'
+
+#env:
+  #- 'AR_VERSION=3.2.17'
+  #- 'AR_VERSION=4.0.3'
+  #- 'AR_VERSION=4.1.0'
+  #- 'AR_VERSION=master'
 
 before_script:
-  - mysql -e 'create database mysql2_test;'
-  - psql -c 'create database postgresql_test;' -U postgres
-  - cd spec/dummy
-  - rake db:migrate RAILS_ENV=mysql2
-  - rake db:migrate RAILS_ENV=postgresql
-  - rake db:migrate RAILS_ENV=sqlite3
-  - cd ../..
+  - cp test/database.yml.example test/database.yml
+  - mysql -e 'CREATE DATABASE mass_insert_test;'
+  - psql -c 'CREATE DATABASE mass_insert_test;' -U postgres

data/Gemfile

@@ -2,11 +2,12 @@ source 'https://rubygems.org'
 
 gemspec
 
-gem "rails", "3.2.13"
-gem "mysql2"
-gem "sqlite3"
-gem "pg"
-gem "rspec"
-gem "simplecov"
-gem "rake"
-gem "coveralls", require: false
+gem 'mysql2'
+gem 'sqlite3'
+gem 'pg'
+
+gem 'simplecov'
+gem 'coveralls', require: false
+
+gem 'rake'
+gem 'activerecord', '>= 3.2'

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013 TODO: Write your name
+Copyright (c) 2014 Alejandro Gutiérrez
 
 MIT License
 

data/README.md

@@ -3,99 +3,55 @@
 This gem aims to provide an easy and faster way to do single database insertions in Rails.
 Support Mysql, PostgreSQL and SQLite3 adapters. It depends on ActiveRecord.
 
-## Installation
 
+## Installation
 Add this line to your application's Gemfile:
+```ruby
+gem 'mass_insert'
+```
+Run the bundle command to install it.
 
-    gem 'mass_insert'
-
-And then execute:
-
-    $ bundle install
-
-Or install it yourself with:
-
-    $ gem install mass_insert
 
 ## Advantages
-
 Faster. It's depending of the computer but these are some results...
-
 * PostgreSQL - Saving 10,000 records in 0.49s
 
+
 ## Attention
+Since this is a single database insertion your model validations will be ignored,
+then if you use this gem you need to be sure that information is OK to be persisted.
 
-Since this is a single database insertion your model validations will be ignored, then if you use this gem you need to be sure that information is OK to be persisted.
 
 ## Basic Usage
+To use MassInsert gem you need to call `mass_insert` method from your ActiveRecord model
+and pass it an array with the values that you want to persist into the database.
+```ruby
+values = [
+  {
+    name:   'Jay',
+    email:  'tremendous_gamer@gmail.com',
+    age:    15
+  },
+  {
+    name:   'Beverly',
+    email:  'nippy_programmer@gmail.com',
+    age:    24
+  }
+]
+
+User.mass_insert(values)
+```
+
+
+## Insertion per batches
+Due you can get a database timeout error you can specify that the insertion will be in batches.
+Just pass the `per_batch` option with the records per batch. Example...
+```ruby
+User.mass_insert(values, per_batch: 1000)
+```
 
-To use MassInsert gem you need to call mass_insert method from your ActiveRecord model and pass it an array with the values that you want to persist into the database.
-
-The array of values:
-
-    values = [
-      {
-        :name   => "Jay",
-        :email  => "tremendous_gamer@gmail.com",
-        :age    => 15
-      },
-      {
-        :name   => "Beverly",
-        :email  => "nippy_programmer@gmail.com",
-        :age    => 24
-      }
-    ]
-
-And call mass_insert method from your model:
-
-    User.mass_insert(values)
-
-
-## Results
-
-Sometimes after MassInsert process you need to see information about the process. MassInsert provides a simple way to do it. Just call the next methods from your model after MassInsert execution.
-
-    User.mass_insert_results.records                    # => 120000
-
-Some result options are...
-
-1. `records` : Returns the amount of records that were persisted.
-2. `time` : Returns the time that took to do all the MassInsert process.
-3. `building_time` : Returns the time that took to create the query string that was persisted.
-4. `execution_time` : Returns the time that took to execute the query string that was persisted.
-
-
-## Options
-
-MassInsert accepts options hash by second param when you call `mass_insert` from your model. These options allow you to configure the way that records will be persisted. Example...
-
-    options = {
-      :some_option => some_value,
-      :some_option => some_value
-    }
-
-    User.mass_insert(values, options)
-
-OR directly
-
-    User.mass_insert(values, :option => value)
-
-Some options you can include are...
-
-**Primary key**
-
-By default primary key is generated automatically. If you wish to set primary key manually you need to pass the `primary_key` option on true. Example...
-
-    User.mass_insert(values, :primary_key => true)
-
-**Each slice**
-
-Due you can get a database timeout error you can specify that the insertion will be in batches. Just pass the `each_slice` option with the records per batch. Example...
-
-    User.mass_insert(values, :each_slice => 10000)
 
 ## Contributing
-
 1. Fork it
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)

data/Rakefile

@@ -1,33 +1,28 @@
-require "bundler/gem_tasks"
+require 'bundler/gem_tasks'
+require 'rake/testtask'
 
-namespace :spec do
-  desc "Runs unit tests"
-  task :unit do
-    system("echo '\e[00;32m\033[1mRunning unit tests...\e[00m'")
-    system("bundle exec rspec spec/lib")
-  end
-
-  adapters = [:mysql2, :postgresql, :sqlite3]
+ADAPTERS = %w(mysql2 postgresql sqlite3)
 
-  adapters.each do |adapter|
-    desc "Runs test suite to #{adapter}"
-    task adapter do
-      ENV["RAILS_ENV"] = adapter.to_s
-      system("echo '\e[00;32m\033[1mRunning test suite to #{adapter}...\e[00m'")
-      system("bundle exec rspec spec/adapters")
+ADAPTERS.each do |adapter|
+  namespace :test do
+    desc "Runs #{adapter} tests."
+    Rake::TestTask.new(adapter) do |t|
+      t.libs << 'test'
+      t.test_files = FileList[
+        "test/support/adapters/#{adapter}.rb",
+        "test/adapters/#{adapter}/**/*_test.rb"]
     end
   end
+end
 
-  desc "Runs all specs"
-  task :all do
-    Rake.application['spec:unit'].invoke
-    raise "Unit test failed!" unless $?.exitstatus == 0
-
-    adapters.each do |adapter|
-      Rake.application["spec:#{adapter}"].invoke
-      raise "Test suite to #{adapter} failed!" unless $?.exitstatus == 0
+namespace :test do
+  desc 'Runs tests.'
+  task :all do |t|
+    ADAPTERS.each do |adapter|
+      Rake.application["test:#{adapter}"].invoke
     end
   end
 end
 
-task default: 'spec:all'
+
+task default: 'test:postgresql'

data/lib/mass_insert/adapters/abstract_adapter.rb

@@ -0,0 +1,54 @@
+module MassInsert
+  module Adapters
+    class AbstractAdapter < SimpleDelegator
+      attr_accessor :values, :options
+
+      def initialize(class_name, values, options)
+        @values  = values
+        @options = options
+        super(class_name)
+      end
+
+      def to_sql
+        "#{insert_sql} #{values_sql}"
+      end
+
+      private
+
+      def columns
+        @columns ||= begin
+          columns = column_names
+          columns.delete(primary_key)
+          columns.map(&:to_sym)
+        end
+      end
+
+      def quoted_columns
+        columns.map do |name|
+          connection.quote_column_name(name)
+        end
+      end
+
+      def insert_sql
+        "INSERT INTO #{quoted_table_name} #{columns_sql} VALUES"
+      end
+
+      def columns_sql
+        "(#{quoted_columns.join(',')})"
+      end
+
+      def values_sql
+        "(#{array_of_attributes_sql.join('),(')});"
+      end
+
+      def array_of_attributes_sql
+        values.map do |attrs|
+          columns.map do |name|
+            value = attrs[name.to_sym] || attrs[name.to_s]
+            connection.quote(value, columns_hash[name.to_s])
+          end.join(',')
+        end
+      end
+    end
+  end
+end

data/lib/mass_insert/adapters/mysql2_adapter.rb

@@ -0,0 +1,5 @@
+module MassInsert
+  module Adapters
+    class Mysql2Adapter < AbstractAdapter; end
+  end
+end

data/lib/mass_insert/adapters/postgresql_adapter.rb

@@ -0,0 +1,5 @@
+module MassInsert
+  module Adapters
+    class PostgreSQLAdapter < AbstractAdapter; end
+  end
+end

data/lib/mass_insert/adapters/sqlite3_adapter.rb

@@ -0,0 +1,5 @@
+module MassInsert
+  module Adapters
+    class SQLite3Adapter < AbstractAdapter; end
+  end
+end

data/lib/mass_insert/base.rb

@@ -1,89 +1,8 @@
 module MassInsert
   module Base
-
-    # = MassInsert
-    #
-    # Invoke mass insert just calling this from your ActiveRecord model...
-    #
-    #   User.mass_insert(values)
-    #
-    # The values should be an array of hashes. Include attributes and values
-    # in every hash. Example...
-    #
-    #   values = [
-    #     {:name   => "user name", :email  => "user email"},
-    #     {:name   => "user name", :email  => "user email"}
-    #   ]
-    #
-    # == Options
-    #
-    # MassInset gem allow you to send it options as second param. Example...
-    #
-    #   User.mass_insert(values, options)
-    #
-    # === Primary key
-    #
-    # By default primary key is ignored. If you wish primary key doesn't
-    # be ignored you need to pass the primary_key option on true.
-    #
-    #   User.mass_insert(values, :primary_key => true)
-    #
-    # === Each slice
-    #
-    # Due you can get a database timeout error you can specify that the
-    # insertion will be in batches. You need to pass the each_slice option
-    # with the records per batch. Example...
-    #
-    #   User.mass_insert(values, :each_slice => 10000)
-    #
-    def mass_insert values, options = {}
-      extend ClassMethods
-
-      options[:class_name]  ||= self
-      options[:each_slice]  ||= false
-      options[:primary_key] ||= false
-
-      @mass_insert_process = Process.new(values, options)
-      @mass_insert_process.start
-    end
-
-
-    module ClassMethods
-
-      # == Results
-      #
-      # Sometimes after MassInsert process you need to see information about
-      # the process. MassInsert provides a simple way to do it. Just call the
-      # next methods from your model after MassInsert execution:
-      #
-      # === Records
-      #
-      # Returns the amount of records that were persisted.
-      #
-      #   User.mass_insert_results.records          # => 120000
-      #
-      # === Time
-      #
-      # Returns the time that took to do all the MassInsert process.
-      #
-      #   User.mass_insert_results.time             # => 0.67
-      #
-      # === Building time
-      #
-      # Returns the time that took to create the query string that was persisted.
-      #
-      #   User.mass_insert_results.building_time    # => 0.58
-      #
-      # === Execution time
-      #
-      # Returns the time that took to execute the query string that was persisted.
-      #
-      #   User.mass_insert_results.execution_time   # => 0.09
-      #
-      def mass_insert_results
-        Result.new(@mass_insert_process)
-      end
-
+    def mass_insert(values, options = {})
+      options[:class_name] ||= self
+      Process.new(values, options).start
     end
   end
 end

data/lib/mass_insert/builder.rb

@@ -0,0 +1,8 @@
+module MassInsert
+  class Builder
+    def build(values, options)
+      class_name = options[:class_name]
+      Utilities.adapter_class.new(class_name, values, options).to_sql
+    end
+  end
+end

data/lib/mass_insert/executer.rb

@@ -1,13 +1,9 @@
 module MassInsert
-  # This class is responsible to execute the queries into the database.
-  # Uses the ActiveRecord::Base.connection.execute functionality.
   class Executer
-
-    def execute queries
-      Array(queries).each do |query|
+    def execute(query)
+      ActiveRecord::Base.transaction do
         ActiveRecord::Base.connection.execute(query)
       end
     end
-
   end
 end

data/lib/mass_insert/process.rb

@@ -1,33 +1,33 @@
-require 'benchmark'
-
 module MassInsert
   class Process
+    attr_reader :values, :options
 
-    def initialize values, options
+    def initialize(values, options)
       @values  = values
       @options = options
     end
 
     def start
-      # MassInsert process is completed by two actions. The first one
-      # gets queries that will be persisted.
-      @building_time  = Benchmark.measure do
-        @queries = builder.build(@values, @options)
-      end
-
-      # The second one executes queries into the database using an
-      # ActiveRecord connection.
-      @execution_time = Benchmark.measure do
-        executer.execute(@queries)
+      ActiveRecord::Base.transaction do
+        values.each_slice(per_batch).each do |batch|
+          query = builder.build(batch, options)
+          executer.execute(query)
+        end
       end
     end
 
+    private
+
     def builder
-      @builder ||= Builder::Base.new
+      @builder ||= Builder.new
     end
 
     def executer
       @executer ||= Executer.new
     end
+
+    def per_batch
+      options[:per_batch] || Utilities.per_batch
+    end
   end
 end

data/lib/mass_insert/utilities.rb

@@ -0,0 +1,25 @@
+module MassInsert
+  class Utilities
+    ADAPTERS = {
+      mysql2:     Adapters::Mysql2Adapter,
+      postgresql: Adapters::PostgreSQLAdapter,
+      sqlite3:    Adapters::SQLite3Adapter,
+    }
+
+    def self.adapter
+      database_config[:adapter].to_sym
+    end
+
+    def self.database_config
+      ActiveRecord::Base.connection.instance_values['config']
+    end
+
+    def self.adapter_class
+      ADAPTERS[Utilities.adapter]
+    end
+
+    def self.per_batch
+      500
+    end
+  end
+end

data/lib/mass_insert/version.rb

@@ -1,3 +1,3 @@
 module MassInsert
-  VERSION = '0.1.3'
+  VERSION = '0.2.0'
 end

data/lib/mass_insert.rb

@@ -1,14 +1,18 @@
+require 'delegate'
+
 module MassInsert
   autoload :Base,        'mass_insert/base.rb'
-  autoload :Result,      'mass_insert/result.rb'
+  autoload :Builder,     'mass_insert/builder.rb'
   autoload :Process,     'mass_insert/process.rb'
   autoload :Executer,    'mass_insert/executer.rb'
+  autoload :Utilities,   'mass_insert/utilities.rb'
   autoload :VERSION,     'mass_insert/version.rb'
 
-  module Builder
-    autoload :Base,      'mass_insert/builder/base.rb'
-    autoload :Adapters,  'mass_insert/builder/adapters.rb'
-    autoload :Utilities, 'mass_insert/builder/utilities.rb'
+  module Adapters
+    autoload :AbstractAdapter,    'mass_insert/adapters/abstract_adapter.rb'
+    autoload :Mysql2Adapter,      'mass_insert/adapters/mysql2_adapter.rb'
+    autoload :PostgreSQLAdapter,  'mass_insert/adapters/postgresql_adapter.rb'
+    autoload :SQLite3Adapter,     'mass_insert/adapters/sqlite3_adapter.rb'
   end
 end
 

data/mass_insert.gemspec

@@ -4,20 +4,20 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'mass_insert/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "mass_insert"
+  spec.name          = 'mass_insert'
   spec.version       = MassInsert::VERSION
-  spec.authors       = ["Alejandro Gutiérrez"]
-  spec.email         = ["alejandrodevs@gmail.com"]
-  spec.summary       = "Mass database insertion in Rails"
-  spec.description   = "This gem aims to provide an easy and faster way to do single database insertions in Rails."
-  spec.homepage      = "https://github.com/alejandrogutierrez/mass_insert"
-  spec.license       = "MIT"
+  spec.authors       = ['Alejandro Gutiérrez']
+  spec.email         = ['alejandrodevs@gmail.com']
+  spec.summary       = 'Mass database insertion in Rails'
+  spec.description   = 'This gem aims to provide an easy and faster way to do single database insertions in Rails.'
+  spec.homepage      = 'https://github.com/alejandrogutierrez/mass_insert'
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files`.split($/)
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake"
+  spec.required_ruby_version = '>= 1.9.3'
+  spec.add_dependency 'activerecord', '>= 3.2'
 end

data/test/adapters/mysql2/example_test.rb

@@ -0,0 +1,3 @@
+require 'test_helper'
+
+shared_examples

data/test/adapters/postgresql/example_test.rb

@@ -0,0 +1,3 @@
+require 'test_helper'
+
+shared_examples

data/test/adapters/sqlite3/example_test.rb

@@ -0,0 +1,3 @@
+require 'test_helper'
+
+shared_examples

data/test/database.yml.example

@@ -0,0 +1,18 @@
+common: &common
+  username: root
+  password:
+  encoding: utf8
+  database: mass_insert_test
+
+mysql2: &mysql2
+  <<: *common
+  adapter: mysql2
+
+postgresql: &postgresql
+  <<: *common
+  username: postgres
+  adapter: postgresql
+
+sqlite3: &sqlite3
+  adapter: sqlite3
+  database: tmp/test.db

data/test/models/user.rb

@@ -0,0 +1,2 @@
+class User < ActiveRecord::Base
+end

data/test/schema.rb

@@ -0,0 +1,11 @@
+ActiveRecord::Schema.define do
+  create_table :users, force: true do |t|
+    t.string  :name
+    t.boolean :active
+    t.integer :age
+    t.decimal :money, precision: 10, scale: 2
+    t.date    :birth_date
+
+    t.timestamps
+  end
+end

data/test/support/adapters/mysql2.rb

@@ -0,0 +1 @@
+ENV['DATABASE_ADAPTER'] = 'mysql2'

data/test/support/adapters/postgresql.rb

@@ -0,0 +1 @@
+ENV['DATABASE_ADAPTER'] = 'postgresql'

data/test/support/adapters/sqlite3.rb

@@ -0,0 +1 @@
+ENV['DATABASE_ADAPTER'] = 'sqlite3'