fast_inserter 0.1.6 → 1.0.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: fe09e32230a3dc433d82f2afe7329cfe6388c46d
-  data.tar.gz: ea29978534d0cb33c67baafa43fbcd1f8d02735a
+SHA256:
+  metadata.gz: 3bbe7d518954dce221c2fecd334c6441f664b9c02041a4c41dc99f231af6801d
+  data.tar.gz: '028e339eb3b132caf9cb397c1c9e13e5505b346654ed6a01146c7e1ec7da5ed0'
 SHA512:
-  metadata.gz: 03556b9df683ce5c6bce7797af5d6754a001167aeae82bd4edc43b2ed2eb0fc21a23c15f32fb740a1fe884a846660ccebda1cc514dd1c048fc6b619b0470ca61
-  data.tar.gz: 389035369e1874a8f522a0c5dbd0be6ddc6e9f16dbf6088101021e05def715d5c204dc49dacb278e8405f4a328c72a7739c472bd3698ada9d233140ed826c0a2
+  metadata.gz: 93828bca03374757e898a731bb33ca9f957f391ef06e1e967ae662874ce49d2a4fdfc75feab2557168299cb22c8bd5ce70d480eeaa9e4ffa0d104a97dda24fef
+  data.tar.gz: efd8eb195e7556e5020edd88341a1f56f1c5f036720e5c4b7ca08dbb5579f56e05a56d651753865d806116bd312fd05b917154f7f36d067c4eabbbd216576a75

data/.ruby-version

@@ -1 +1 @@
-2.3.0
+2.5.3

data/.travis.yml

@@ -3,9 +3,9 @@ sudo: required
 notifications:
   email: false
 rvm:
-  - 2.3.0
-  - 2.4.4
-  - 2.5.1
+  - 2.3.8
+  - 2.4.5
+  - 2.5.3
   - ruby-head
 env:
   matrix:

data/CHANGELOG.md

@@ -1,4 +1,14 @@
-## Unreleased ##
+## 1.0.0.pre (January 7, 2019) ##
+
+*   Use database IN clause when checking for existing records.
+
+    *Jordon Dornbos*
+
+## 0.1.6 (July 3, 2018) ##
+
+*   Fix check_for_existing with multiple variable columns.
+
+    *Josh Warfield*
 
 *   Remove support for Ruby 2.2. Add support for Ruby 2.4 and 2.5.
 

data/README.md

@@ -51,9 +51,9 @@ params = {
   options: {
     timestamps: true,
     unique: true,
-    check_for_existing: true,
-    group_size: 2_000
+    check_for_existing: true
   },
+  group_size: 2_000,
   variable_column: 'user_id',
   values: user_ids
 }
@@ -89,7 +89,7 @@ Queries the table for any values which already exist and removes them from the v
 
 ### group_size
 
-Insertions will be broken up into batches. This specifies the number of records you want to insert per batch. Default is 2,000.
+Insertions will be broken up into batches. This specifies the number of records you want to insert per batch. Default is 1,000.
 
 ### variable_column
 

data/fast_inserter.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'activerecord', '>= 4.1.0'
 
-  spec.required_ruby_version = ">= 2.2.0"
+  spec.required_ruby_version = ">= 2.3.8"
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/fast_inserter/fast_inserter_base.rb

@@ -18,8 +18,7 @@
 #     options: {
 #       timestamps: true,
 #       unique: true,
-#       check_for_existing: true,
-#       group_size: 1_000
+#       check_for_existing: true
 #     },
 #     variable_column: 'user_id',
 #     values: user_ids
@@ -39,11 +38,11 @@
 #     A hash representing additional column values to set that you don't want
 #     to include in uniqueness checks or other pre-insertion operations.
 #   group_size: Integer
-#     The number of items you want to insert per batch of records. Default 10_000.
+#     The number of items you want to insert per batch of records. Defaults to 1_000.
 #
 module FastInserter
   class Base
-    DEFAULT_GROUP_SIZE = 2_000
+    DEFAULT_GROUP_SIZE = 1_000
 
     def initialize(params)
       @table_name = params[:table]
@@ -92,20 +91,33 @@ module FastInserter
       end
     end
 
+    def existing_values_sql(group_of_values)
+      sql = "SELECT #{@variable_columns.join(', ')} FROM #{@table_name} WHERE #{values_hash_to_sql(@static_columns)}"
+
+      if @variable_columns.length > 1
+        group_of_values_sql = group_of_values.map do |group|
+          values_hash = variable_column_values_to_hash(group)
+          "(#{values_hash_to_sql(values_hash)})"
+        end.join(' OR ')
+
+        sql += " AND (#{group_of_values_sql})"
+      else
+        values_to_check = ActiveRecord::Base.send(:sanitize_sql_array, ['?', group_of_values.flatten])
+        sql += " AND #{@variable_columns.first} IN (#{values_to_check})"
+      end
+
+      sql
+    end
+
     # Queries for the existing values for a given group of values
     def existing_values(group_of_values)
-      sql = "SELECT #{@variable_columns.join(', ')} FROM #{@table_name} WHERE #{existing_values_static_columns}"
+      sql = existing_values_sql(group_of_values)
 
       # NOTE: There are more elegant ways to get this field out of the resultset, but each database adaptor returns a different type
       # of result from 'execute(sql)'. Potential classes for 'result' is Array (sqlite), Mysql2::Result (mysql2), PG::Result (pg). Each
       # result can be enumerated into a list of arrays (mysql) or list of hashes (sqlite, pg)
       results = ActiveRecord::Base.connection.execute(sql)
-      existing_values = stringify_values(results)
-
-      # Rather than a giant IN query in the sql statement (which can be bad for database performance),
-      # do the filtering of relevant values here in a ruby select.
-      group_of_values_strings = stringify_values(group_of_values)
-      existing_values & group_of_values_strings
+      stringify_values(results)
     end
 
     def stringify_values(results)
@@ -118,8 +130,18 @@ module FastInserter
       end
     end
 
-    def existing_values_static_columns
-      @static_columns.map do |key, value|
+    def variable_column_values_to_hash(values)
+      hash = {}
+
+      @variable_columns.each_with_index do |variable_column, index|
+        hash[variable_column] = values[index]
+      end
+
+      hash
+    end
+
+    def values_hash_to_sql(values)
+      values.map do |key, value|
         if value.nil?
           "#{key} IS NULL"
         else

data/lib/fast_inserter/version.rb

@@ -1,3 +1,3 @@
 module FastInserter
-  VERSION = "0.1.6"
+  VERSION = '1.0.0.pre'.freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fast_inserter
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 1.0.0.pre
 platform: ruby
 authors:
 - Scott Ringwelski
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-03 00:00:00.000000000 Z
+date: 2019-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -137,15 +137,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.0
+      version: 2.3.8
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Quickly insert database records in bulk