fast_inserter 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4e0df4e509d7f47f48a2d9ef2616c91586bc295f
+  data.tar.gz: 06523b7bc59f458a99e57824a92e41f5d9b6722a
+SHA512:
+  metadata.gz: e63d8e35019b8de1cb16861e82350ce292fed7e4d9293738b53d40ada215c5ff16fe77e3c8daa7ba573f3b7758782166eac7d4f688686f45a0b2946acd5e7b32
+  data.tar.gz: a7c7e1264a29a90ed2d9ed4e52e222843e564bd67a161f7727fd8f5f9c422be27d1d1f33fd9227658b44a55d03cc2d4b1d9a284a7c35bdf1d0d90316f12cd22e

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.ruby-version

@@ -0,0 +1 @@
+2.3.0

data/.travis.yml

@@ -0,0 +1,21 @@
+language: ruby
+sudo: required
+notifications:
+  email: false
+rvm:
+  - 2.2.4
+  - 2.3.0
+  - ruby-head
+env:
+  matrix:
+    - DB=pg
+    - DB=mysql
+    - DB=sqlite
+before_install:
+  # Sqlite set up involves installing a more modern version which we move into a script.
+  - sh -c "if [ '$DB' = 'sqlite' ]; then ./ci/install_modern_sqlite.sh; fi;"
+before_script:
+  # PG and mysql is simpler.
+  - sh -c "if [ '$DB' = 'pg' ]; then psql -c 'DROP DATABASE IF EXISTS fast_inserter;' -U postgres; fi"
+  - sh -c "if [ '$DB' = 'pg' ]; then psql -c 'create database fast_inserter;' -U postgres; fi"
+  - sh -c "if [ '$DB' = 'mysql' ]; then mysql -e 'create database IF NOT EXISTS fast_inserter;'; fi"

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fast_inserter.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Stryder Corp
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,113 @@
+# FastInserter
+
+[![Build Status](https://travis-ci.org/strydercorp/fast_inserter.svg?branch=master)](https://travis-ci.org/strydercorp/fast_inserter)
+
+Use raw SQL to insert database records in bulk. Supports uniqueness constraints, timestamps, and checking for existing records.
+
+The motivation for this library from the fact that rails does validations on each and every inserted record in the join table. And, even if you pass validate: false, it still loads each record and inserts one by one. This leads to very slow insertion of large number (thoasands) of records.
+
+This library skips active record altogether and uses raw sql to insert records. However, using raw sql goes around all your business logic, so we provide ways to still have niceties like uniqueness constraints and timestamps.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fast_inserter'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fast_inserter
+
+## Runtime dependencies
+
+* activerecord: Fast inserter depends on active record for handling database connections, database configuration, executing the sql, and sql sanitization.
+
+## Usage
+
+In most cases, you probably don't want to use this library and instead should active record. However, should you need to use this library, usage instructions are below.
+
+A basic usage for inserting multiple 'MassEmailsUser' records:
+
+```ruby
+@mass_email = MassEmail.find(params[:id])
+user_ids = [1, 2, 3, 4] # ids to fast insert
+params = {
+  table: 'mass_emails_users',
+  static_columns: {
+    mass_email_id: @mass_email.id
+  },
+  additional_columns: {
+    created_by_id: current_user.id
+  },
+  options: {
+    timestamps: true,
+    unique: true,
+    check_for_existing: true,
+    group_size: 10_000
+  },
+  variable_column: 'user_id',
+  values: user_ids
+}
+inserter = FastInserter::Base.new(params)
+inserter.fast_insert
+```
+
+Let's walkthrough the options.
+
+### table
+
+Defines the table name to insert into
+
+### static_columns
+
+These are columns and the values for the columns which will not change for each insertion.
+
+### additional_columns
+
+These are also static columns which will not changed, but these columns will not be used for uniqueness validation constraints.
+
+### timestamps
+
+Includes created_at and updated_at timestamps to each record. Default is false.
+
+### unique
+
+Ensures that the 'values' parameter is a unique set of values. Default is false.
+
+### check_for_existing
+
+Queries the table for any values which already exist and removes them from the values to be inserted. This query uses 'static_columns' and 'variable_column' for determining uniqueness. Default is false.
+
+### group_size
+
+Insertions will be broken up into batches. This specifies the number of records you want to insert per batch. Default is 10,000.
+
+### variable_column
+
+The name of the column which we will be dynamically inserting records for. This is the only column which changes per-record being inserted.
+
+### values
+
+The large list of values to use for the 'variable_column' value when inserting the records.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/strydercorp/fast_inserter. All code must run on sqlite, pg, and mysql (tests are set up CI already).
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "fast_inserter"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/ci/install_modern_sqlite.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e # halt script on error
+
+sudo apt-get autoremove sqlite3
+sudo apt-get install python-software-properties
+sudo apt-add-repository -y ppa:travis-ci/sqlite3
+sudo apt-get -y update
+sudo apt-cache show sqlite3
+sudo apt-get install sqlite3=3.7.15.1-1~travis1
+sudo sqlite3 -version
+sudo psql --version
+sudo mysql --version
+gem update bundler

data/fast_inserter.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fast_inserter/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fast_inserter"
+  spec.version       = FastInserter::VERSION
+  spec.authors       = ["Scott Ringwelski", "Brandon Gafford", "Jordon Dornbos"]
+  spec.email         = ["scott@joinhandshake.com", "brandon@joinhandshake.com", "jordon@joinhandshake.com"]
+
+  spec.summary       = %q{Quickly insert database records in bulk}
+  spec.description   = %q{Use raw SQL to insert database records in bulk. Supports uniqueness constraints, timestamps, and checking for existing records.}
+  spec.homepage      = "https://github.com/strydercorp/fast_inserter."
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'activerecord', '>= 4.1.0'
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "database_cleaner"
+
+  case ENV['DB']
+  when "mysql"; spec.add_development_dependency "mysql2"
+  when "sqlite"; spec.add_development_dependency "sqlite3"
+  when "pg"; spec.add_development_dependency "pg"
+  else spec.add_development_dependency "sqlite3" # Default
+  end
+end

data/lib/fast_inserter/fast_inserter_base.rb

@@ -0,0 +1,158 @@
+# Highly based off of https://github.com/sportngin/m2m_fast_insert
+# Unfortunately, that gem was not up to date for rails 4.
+#
+# NOTE:
+#   - Only 'static_column' values and 'values' are sanitized for SQL injection.
+#
+# Example usage:
+#   @mass_email = MassEmail.find(params[:id])
+#   user_ids = [1, 2, 3, 4] # ids to fast insert
+#   params = {
+#     table: 'mass_emails_users',
+#     static_columns: {
+#       mass_email_id: @mass_email.id
+#     },
+#     additional_columns: {
+#       created_by_id: current_user.id
+#     },
+#     options: {
+#       timestamps: true,
+#       unique: true,
+#       check_for_existing: true,
+#       group_size: 1_000
+#     },
+#     variable_column: 'user_id',
+#     values: user_ids
+#   }
+#   inserter = FastInserter::Base.new(params)
+#   inserter.fast_insert
+#
+# Options:
+#   unique: true
+#     Makes sure all values are unique
+#   check_for_existing: true
+#     Checks if values already exist in the database and only inserts nonexisting values
+#     This checks values scoped to static columns.
+#   timestamps: true
+#     Adds created_at and updated_at columns to insert statement
+#   additional_columns: Hash
+#     A hash representing additional column values to set that you don't want
+#     to include in uniqueness checks or other pre-insertion operations.
+#   group_size: Integer
+#     The number of items you want to insert per batch of records. Default 10_000.
+#
+module FastInserter
+  class Base
+    def initialize(params)
+      @table_name = params[:table]
+      @static_columns = params[:static_columns]
+      @additional_columns = params[:additional_columns]
+      @variable_column = params[:variable_column]
+      @options = params[:options] || {}
+
+      # We want to break up the insertions into multiple transactiosn in case there
+      # is a very large amount of values. This avoids PG:OutOfMemory errors and smooths
+      # out the load. The second 'false' param means don't fill in the last group with nil elements.
+      all_values = params[:values]
+      all_values.uniq! if @options[:unique]
+      group_size = Integer(params[:group_size] || 10_000)
+      @value_groups = all_values.in_groups_of(group_size, false)
+    end
+
+    # Iterates through the value groups (which is all values in groups of smaller sizes)
+    # and generates and executes a transaction to insert those groups one at a time
+    def fast_insert
+      return if nothing_to_insert?
+
+      @value_groups.each do |group|
+        fast_insert_group(group)
+      end
+    end
+
+  private
+
+    def nothing_to_insert?
+      @value_groups.empty?
+    end
+
+    # For a given group of IDS, generates the transaction to execute and does. If we want to check existing
+    # we make sure that we check the existing within the same transaction.
+    def fast_insert_group(group)
+      if @options[:check_for_existing]
+        ActiveRecord::Base.transaction do
+          non_existing_values = group.map(&:to_s) - existing_values(group)
+          sql_string = insertion_sql_for_group(non_existing_values)
+          ActiveRecord::Base.connection.execute(sql_string) unless non_existing_values.empty?
+        end
+      else
+        sql_string = insertion_sql_for_group(group)
+        ActiveRecord::Base.connection.execute(sql_string)
+      end
+    end
+
+    # Queries for the existing values for a given group of values
+    def existing_values(group_of_values)
+      values_to_check = ActiveRecord::Base.send(:sanitize_sql_array, ["?", group_of_values])
+      sql = "SELECT #{@variable_column} FROM #{@table_name} WHERE #{existing_values_static_columns} AND #{@variable_column} IN (#{values_to_check})"
+
+      # NOTE: There are more elegant ways to get this field out of the resultset, but each database adaptor returns a different type
+      # of result from 'execute(sql)'. Potential classes for 'result' is Array (sqlite), Mysql2::Result (mysql2), PG::Result (pg). Each
+      # result can be enumerated into a list of arrays (mysql) or list of hashes (sqlite, pg)
+      results = ActiveRecord::Base.connection.execute(sql)
+      results.to_a.map do |result|
+        if result.is_a?(Hash)
+          result[@variable_column].to_s
+        elsif result.is_a?(Array)
+          result[0].to_s
+        end
+      end
+    end
+
+    def existing_values_static_columns
+      @static_columns.map do |key, value|
+        sanitized_value = ActiveRecord::Base.send(:sanitize_sql_array, ["?", value])
+        "#{key} = #{sanitized_value}"
+      end.join(' AND ')
+    end
+
+    def insertion_sql_for_group(group_of_values)
+      "INSERT INTO #{@table_name} (#{column_names}) VALUES #{insert_values(group_of_values)}"
+    end
+
+    def column_names
+      "#{all_static_columns.keys.join(', ')}, #{@variable_column}"
+    end
+
+    def all_static_columns
+      @all_static_columns ||= begin
+        rv = @static_columns.dup
+
+        if @options[:timestamps]
+          time = Time.now
+          rv[:created_at] = time
+          rv[:updated_at] = time
+        end
+
+        if @additional_columns.present?
+          @additional_columns.each do |key, value|
+            rv[key] = value
+          end
+        end
+
+        rv
+      end
+    end
+
+    def insert_values(group_of_values)
+      rv = []
+      static_column_values = ActiveRecord::Base.send(:sanitize_sql_array, ["?", all_static_columns.values])
+
+      group_of_values.each do |value|
+        value = ActiveRecord::Base.send(:sanitize_sql_array, ["?", value])
+        rv << "(#{static_column_values},#{value})"
+      end
+
+      rv.join(', ')
+    end
+  end
+end

data/lib/fast_inserter/version.rb

@@ -0,0 +1,3 @@
+module FastInserter
+  VERSION = "0.1.0"
+end

data/lib/fast_inserter.rb

@@ -0,0 +1,5 @@
+require "fast_inserter/fast_inserter_base"
+require "fast_inserter/version"
+
+module FastInserter
+end

metadata

@@ -0,0 +1,148 @@
+--- !ruby/object:Gem::Specification
+name: fast_inserter
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Scott Ringwelski
+- Brandon Gafford
+- Jordon Dornbos
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-02-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Use raw SQL to insert database records in bulk. Supports uniqueness constraints,
+  timestamps, and checking for existing records.
+email:
+- scott@joinhandshake.com
+- brandon@joinhandshake.com
+- jordon@joinhandshake.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- ci/install_modern_sqlite.sh
+- fast_inserter.gemspec
+- lib/fast_inserter.rb
+- lib/fast_inserter/fast_inserter_base.rb
+- lib/fast_inserter/version.rb
+homepage: https://github.com/strydercorp/fast_inserter.
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Quickly insert database records in bulk
+test_files: []