bulk_insert 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a4e0d183ca8739d8b1fc3a49f9fa202a2e680ab9
+  data.tar.gz: c5e41cf70f177ddd5d0dd8238c82cb037c9f0055
+SHA512:
+  metadata.gz: 53ceccf9f12ca48381a924f6b7d72b398003b88e17c364700c85293e309cc2db8aa6710db51312614fff4fa9cd1691e8dd5fc089fa55de1c499ab1099716a47b
+  data.tar.gz: da31d80b39821b97292d5da6a1ca2f1df6b9b6ced154637ce81472c2fdecb6aa4fc623ec64b5ee15470ac9f0a9f6195bb4261fc0d9940c436931aae9c73a43b7

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2015 Jamis Buck
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,96 @@
+# BulkInsert
+
+A little ActiveRecord extension for helping to insert lots of rows in a
+single insert statement.
+
+## Installation
+
+Add it to your Gemfile:
+
+    gem 'bulk_insert'
+
+## Usage
+
+BulkInsert adds a new class method to your ActiveRecord models:
+
+    class Book < ActiveRecord::Base
+    end
+
+    book_attrs = ... # some array of hashes, for instance
+    Book.bulk_insert do |worker|
+      book_attrs.each do |attrs|
+        worker.add(attrs)
+      end
+    end
+
+All of those `#add` calls will be accumulated into a single SQL insert
+statement, vastly improving the performance of multiple sequential
+inserts (think data imports and the like).
+
+By default, the columns to be inserted will be all columns in the table,
+minus the `id` column, but if you want, you can explicitly enumerate
+the columns:
+
+    Book.bulk_insert(:title, :author) do |worker|
+      # specify a row as an array of values...
+      worker.add ["Eye of the World", "Robert Jordan"]
+
+      # or as a hash
+      worker.add title: "Lord of Light", author: "Roger Zelazny"
+    end
+
+It will automatically set `created_at`/`updated_at` columns to the current
+date, as well.
+
+    Book.bulk_insert(:title, :author, :created_at, :updated_at) do |worker|
+      # specify created_at/updated_at explicitly...
+      worker.add ["The Chosen", "Chaim Potok", Time.now, Time.now]
+
+      # or let BulkInsert set them by default...
+      worker.add ["Hello Ruby", "Linda Liukas"]
+    end
+
+By default, the batch is always saved when the block finishes, but you
+can explicitly save inside the block whenever you want, by calling
+`#save!` on the worker:
+
+    Book.bulk_insert do |worker|
+      worker.add(...)
+      worker.add(...)
+
+      worker.save!
+
+      worker.add(...)
+      #...
+    end
+
+That will save the batch as it has been defined to that point, and then
+empty the batch so that you can add more rows to it if you want.
+
+
+### Batch Set Size
+
+By default, the size of the insert is limited to 500 rows at a time.
+This is called the _set size_. If you add another row that causes the
+set to exceed the set size, the insert statement is automatically built
+and executed, and the batch is reset.
+
+If you want a larger (or smaller) set size, you can specify it in
+two ways:
+
+    # specify set_size when initializing the bulk insert...
+    Book.bulk_insert(set_size: 100) do |worker|
+      # ...
+    end
+
+    # specify it on the worker directly...
+    Book.bulk_insert do |worker|
+      worker.set_size = 100
+      # ...
+    end
+
+
+## License
+
+BulkInsert is released under the MIT license (see MIT-LICENSE) by
+Jamis Buck (jamis@jamisbuck.org).

data/Rakefile

@@ -0,0 +1,34 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'BulkInsert'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+
+
+
+
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+
+task default: :test

data/lib/bulk_insert.rb

@@ -0,0 +1,24 @@
+require 'bulk_insert/worker'
+
+module BulkInsert
+  extend ActiveSupport::Concern
+
+  class_methods do
+    def bulk_insert(*columns, set_size:500)
+      columns = self.column_names - %w(id) if columns.empty?
+      worker = BulkInsert::Worker.new(connection, table_name, columns, set_size)
+
+      if block_given?
+        transaction do
+          yield worker
+          worker.save!
+        end
+        self
+      else
+        worker
+      end
+    end
+  end
+end
+
+ActiveRecord::Base.send :include, BulkInsert

data/lib/bulk_insert/version.rb

@@ -0,0 +1,7 @@
+module BulkInsert
+  MAJOR = 1
+  MINOR = 0
+  TINY  = 0
+
+  VERSION = [MAJOR, MINOR, TINY].join(".")
+end

data/lib/bulk_insert/worker.rb

@@ -0,0 +1,64 @@
+module BulkInsert
+  class Worker
+    attr_reader :connection
+    attr_accessor :set_size
+
+    def initialize(connection, table_name, column_names, set_size=500)
+      @connection = connection
+      @set_size = set_size
+
+      columns = connection.columns(table_name)
+      column_map = columns.inject({}) { |h, c| h.update(c.name => c) }
+
+      @columns = column_names.map { |name| column_map[name.to_s] }
+      @table_name = connection.quote_table_name(table_name)
+      @column_names = column_names.map { |name| connection.quote_column_name(name) }.join(",")
+
+      @set = []
+    end
+
+    def pending?
+      @set.any?
+    end
+
+    def add(values)
+      save! if @set.length >= set_size
+
+      values = values.with_indifferent_access if values.is_a?(Hash)
+      mapped = @columns.map.with_index do |column, index|
+          value = values.is_a?(Hash) ? values[column.name] : values[index]
+          if value.nil?
+            if column.name == "created_at" || column.name == "updated_at"
+              Time.now
+            else
+              value
+            end
+          else
+            value
+          end
+        end
+
+      @set.push(mapped)
+    end
+
+    def save!
+      if pending?
+        sql = "INSERT INTO #{@table_name} (#{@column_names}) VALUES "
+
+        rows = []
+        @set.each do |row|
+          values = []
+          @columns.zip(row) do |column, value|
+            values << @connection.quote(value, column)
+          end
+          rows << "(#{values.join(',')})"
+        end
+
+        sql << rows.join(",")
+        @connection.execute(sql)
+
+        @set.clear
+      end
+    end
+  end
+end

data/test/bulk_insert/worker_test.rb

@@ -0,0 +1,84 @@
+require 'test_helper'
+
+class BulkInsertWorkerTest < ActiveSupport::TestCase
+  setup do
+    @insert = BulkInsert::Worker.new(
+      Testing.connection,
+      Testing.table_name,
+      %w(greeting age happy created_at updated_at))
+    @now = Time.now
+  end
+
+  test "empty insert is not pending" do
+    assert_equal false, @insert.pending?
+  end
+
+  test "default set size" do
+    assert_equal 500, @insert.set_size
+  end
+
+  test "adding row to insert makes insert pending" do
+    @insert.add ["Hello", 15, true, @now, @now]
+    assert_equal true, @insert.pending?
+  end
+
+  test "add should default timestamp columns to current time" do
+    now = Time.now
+
+    @insert.add ["Hello", 15, true]
+    @insert.save!
+
+    record = Testing.first
+    assert_operator record.created_at, :>=, now
+    assert_operator record.updated_at, :>=, now
+  end
+
+  test "add should allow values given as Hash" do
+    @insert.add greeting: "Yo", age: 20, happy: false, created_at: @now, updated_at: @now
+    @insert.save!
+
+    record = Testing.first
+    assert_not_nil record
+    assert_equal "Yo", record.greeting
+    assert_equal 20, record.age
+    assert_equal false, record.happy?
+  end
+
+  test "add should save automatically when overflowing set size" do
+    @insert.set_size = 1
+    @insert.add ["Hello", 15, true, @now, @now]
+    @insert.add ["Yo", 20, false, @now, @now]
+    assert_equal 1, Testing.count
+    assert_equal "Hello", Testing.first.greeting
+  end
+
+  test "save! makes insert not pending" do
+    @insert.add ["Hello", 15, true, @now, @now]
+    @insert.save!
+    assert_equal false, @insert.pending?
+  end
+
+  test "save! when not pending should do nothing" do
+    assert_no_difference 'Testing.count' do
+      @insert.save!
+    end
+  end
+
+  test "save! inserts pending records" do
+    @insert.add ["Yo", 15, false, @now, @now]
+    @insert.add ["Hello", 25, true, @now, @now]
+    @insert.save!
+
+    yo = Testing.find_by(greeting: 'Yo')
+    hello = Testing.find_by(greeting: 'Hello')
+
+    assert_not_nil yo
+    assert_equal 15, yo.age
+    assert_equal false, yo.happy?
+
+    assert_not_nil hello
+    assert_equal 25, hello.age
+    assert_equal true, hello.happy?
+  end
+end
+

data/test/bulk_insert_test.rb

@@ -0,0 +1,22 @@
+require 'test_helper'
+
+class BulkInsertTest < ActiveSupport::TestCase
+  test "bulk_insert without block should return worker" do
+    result = Testing.bulk_insert
+    assert_kind_of BulkInsert::Worker, result
+  end
+
+  test "bulk_insert with block should yield worker" do
+    result = nil
+    Testing.bulk_insert { |worker| result = worker }
+    assert_kind_of BulkInsert::Worker, result
+  end
+
+  test "bulk_insert with block should save automatically" do
+    assert_difference "Testing.count", 1 do
+      Testing.bulk_insert do |worker|
+        worker.add greeting: "Hello"
+      end
+    end
+  end
+end

data/test/dummy/README.rdoc

@@ -0,0 +1,28 @@
+== README
+
+This README would normally document whatever steps are necessary to get the
+application up and running.
+
+Things you may want to cover:
+
+* Ruby version
+
+* System dependencies
+
+* Configuration
+
+* Database creation
+
+* Database initialization
+
+* How to run the test suite
+
+* Services (job queues, cache servers, search engines, etc.)
+
+* Deployment instructions
+
+* ...
+
+
+Please feel free to use a different markup language if you do not plan to run
+<tt>rake doc:app</tt>.

data/test/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require File.expand_path('../config/application', __FILE__)
+
+Rails.application.load_tasks

data/test/dummy/app/assets/javascripts/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/test/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any styles
+ * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
+ * file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/test/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,5 @@
+class ApplicationController < ActionController::Base
+  # Prevent CSRF attacks by raising an exception.
+  # For APIs, you may want to use :null_session instead.
+  protect_from_forgery with: :exception
+end

data/test/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end