fast_inserter 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 120b53eb90d8eef25a9d1f3340b36f4256d9ddca
-  data.tar.gz: aea33ef6b99f3707b2276785cc5add4d74522020
+  metadata.gz: fe09e32230a3dc433d82f2afe7329cfe6388c46d
+  data.tar.gz: ea29978534d0cb33c67baafa43fbcd1f8d02735a
 SHA512:
-  metadata.gz: e47da657ffab5623bae32686880e916f507b49bd9a33067be2c80ee1131bf269430dc7a2a670045acd478d5a0a16f9d7c1afd971fc1aace6b96db19b89a7d102
-  data.tar.gz: b694592a8c016b52f39993cd3695316d383868f6cf267b84f50682e4eab662ab5f123a0b041ef1f3759b9d7c620593cb9fb83a98dd7e23c2cbc63bf658779780
+  metadata.gz: 03556b9df683ce5c6bce7797af5d6754a001167aeae82bd4edc43b2ed2eb0fc21a23c15f32fb740a1fe884a846660ccebda1cc514dd1c048fc6b619b0470ca61
+  data.tar.gz: 389035369e1874a8f522a0c5dbd0be6ddc6e9f16dbf6088101021e05def715d5c204dc49dacb278e8405f4a328c72a7739c472bd3698ada9d233140ed826c0a2

data/.travis.yml

@@ -3,17 +3,15 @@ sudo: required
 notifications:
   email: false
 rvm:
-  - 2.2.4
   - 2.3.0
+  - 2.4.4
+  - 2.5.1
   - ruby-head
 env:
   matrix:
     - DB=pg
     - DB=mysql
     - DB=sqlite
-before_install:
-  # Sqlite set up involves installing a more modern version which we move into a script.
-  - sh -c "if [ '$DB' = 'sqlite' ]; then ./ci/install_modern_sqlite.sh; fi;"
 before_script:
   # PG and mysql is simpler.
   - sh -c "if [ '$DB' = 'pg' ]; then psql -c 'DROP DATABASE IF EXISTS fast_inserter;' -U postgres; fi"

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## Unreleased ##
+
+*   Remove support for Ruby 2.2. Add support for Ruby 2.4 and 2.5.
+
+    *Scott Ringwelski*
+
+## 0.1.5 (January 9, 2017) ##
+
+*   Support multiple variable columns
+
+    *Josh Warfield*
+
 ## 0.1.4 (September 19, 2016) ##
 
 *   Improve performance when checking for existing records.

data/README.md

@@ -1,7 +1,7 @@
 # FastInserter
 
 [![Gem Version](https://badge.fury.io/rb/fast_inserter.svg)](https://badge.fury.io/rb/fast_inserter)
-[![Build Status](https://travis-ci.org/strydercorp/fast_inserter.svg?branch=master)](https://travis-ci.org/strydercorp/fast_inserter)
+[![Build Status](https://travis-ci.org/joinhandshake/fast_inserter.svg?branch=master)](https://travis-ci.org/joinhandshake/fast_inserter)
 
 Use raw SQL to insert database records in bulk, fast. Supports uniqueness constraints, timestamps, and checking for existing records.
 
@@ -52,7 +52,7 @@ params = {
     timestamps: true,
     unique: true,
     check_for_existing: true,
-    group_size: 10_000
+    group_size: 2_000
   },
   variable_column: 'user_id',
   values: user_ids
@@ -89,7 +89,7 @@ Queries the table for any values which already exist and removes them from the v
 
 ### group_size
 
-Insertions will be broken up into batches. This specifies the number of records you want to insert per batch. Default is 10,000.
+Insertions will be broken up into batches. This specifies the number of records you want to insert per batch. Default is 2,000.
 
 ### variable_column
 
@@ -116,7 +116,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/strydercorp/fast_inserter. All code must run on sqlite, pg, and mysql (tests are set up CI already).
+Bug reports and pull requests are welcome on GitHub at https://github.com/joinhandshake/fast_inserter. All code must run on sqlite, pg, and mysql (tests are set up CI already).
 
 
 ## License

data/fast_inserter.gemspec

@@ -6,8 +6,8 @@ require 'fast_inserter/version'
 Gem::Specification.new do |spec|
   spec.name          = "fast_inserter"
   spec.version       = FastInserter::VERSION
-  spec.authors       = ["Scott Ringwelski", "Brandon Gafford", "Jordon Dornbos", "Matt Hickman"]
-  spec.email         = ["scott@joinhandshake.com", "brandon@joinhandshake.com", "jordon@joinhandshake.com", "matt@joinhandshake.com"]
+  spec.authors       = ["Scott Ringwelski", "Brandon Gafford", "Jordon Dornbos", "Matt Hickman", "Josh Warfield"]
+  spec.email         = ["scott@joinhandshake.com", "brandon@joinhandshake.com", "jordon@joinhandshake.com", "matt@joinhandshake.com", "josh@joinhandshake.com"]
 
   spec.summary       = %q{Quickly insert database records in bulk}
   spec.description   = %q{Use raw SQL to insert database records in bulk. Supports uniqueness constraints, timestamps, and checking for existing records.}

data/lib/fast_inserter/fast_inserter_base.rb

@@ -82,7 +82,7 @@ module FastInserter
     def fast_insert_group(group)
       if @options[:check_for_existing]
         ActiveRecord::Base.transaction do
-          non_existing_values = group.map { |values| values.map(&:to_s) } - existing_values(group)
+          non_existing_values = stringify_values(group) - existing_values(group)
           sql_string = insertion_sql_for_group(non_existing_values)
           ActiveRecord::Base.connection.execute(sql_string) unless non_existing_values.empty?
         end
@@ -100,20 +100,24 @@ module FastInserter
       # of result from 'execute(sql)'. Potential classes for 'result' is Array (sqlite), Mysql2::Result (mysql2), PG::Result (pg). Each
       # result can be enumerated into a list of arrays (mysql) or list of hashes (sqlite, pg)
       results = ActiveRecord::Base.connection.execute(sql)
-      existing_values = results.to_a.map do |result|
-        if result.is_a?(Hash)
-          @variable_columns.map { |col| result[col] }.map(&:to_s)
-        elsif result.is_a?(Array)
-          result.map(&:to_s)
-        end
-      end
+      existing_values = stringify_values(results)
 
       # Rather than a giant IN query in the sql statement (which can be bad for database performance),
       # do the filtering of relevant values here in a ruby select.
-      group_of_values_strings = group_of_values.map { |values| values.map(&:to_s) }
+      group_of_values_strings = stringify_values(group_of_values)
       existing_values & group_of_values_strings
     end
 
+    def stringify_values(results)
+      results.to_a.map do |result|
+        if result.is_a?(Hash)
+          @variable_columns.map { |col| ActiveRecord::Base.connection.type_cast(result[col], column_definitions[col]) }
+        elsif result.is_a?(Array)
+          result.map.with_index { |val, i| ActiveRecord::Base.connection.type_cast(val, column_definitions[@variable_columns[i]]) }
+        end
+      end
+    end
+
     def existing_values_static_columns
       @static_columns.map do |key, value|
         if value.nil?
@@ -164,5 +168,13 @@ module FastInserter
 
       rv.join(', ')
     end
+
+    def column_definitions
+      @column_definitions ||= begin
+        ActiveRecord::Base.connection.columns(@table_name).reduce({}) do |memo, column|
+          memo.merge!(column.name => column)
+        end
+      end
+    end
   end
 end

data/lib/fast_inserter/version.rb

@@ -1,3 +1,3 @@
 module FastInserter
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: fast_inserter
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Scott Ringwelski
 - Brandon Gafford
 - Jordon Dornbos
 - Matt Hickman
+- Josh Warfield
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-01-09 00:00:00.000000000 Z
+date: 2018-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -104,6 +105,7 @@ email:
 - brandon@joinhandshake.com
 - jordon@joinhandshake.com
 - matt@joinhandshake.com
+- josh@joinhandshake.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -119,7 +121,6 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
-- ci/install_modern_sqlite.sh
 - fast_inserter.gemspec
 - lib/fast_inserter.rb
 - lib/fast_inserter/fast_inserter_base.rb

data/ci/install_modern_sqlite.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -e # halt script on error
-
-sudo apt-get autoremove sqlite3
-sudo apt-get install python-software-properties
-sudo apt-add-repository -y ppa:travis-ci/sqlite3
-sudo apt-get -y update
-sudo apt-cache show sqlite3
-sudo apt-get install sqlite3=3.7.15.1-1~travis1
-sudo sqlite3 -version
-sudo psql --version
-sudo mysql --version
-gem update bundler