fast_inserter 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 73fb3eacb5a0e1b3bbca479f5d900092530dd5a4
-  data.tar.gz: d10d296537aa9f02f6db25876a26546ce22de053
+  metadata.gz: c9a5654ca484c5f0b47bfa91c6f5e1bb52549933
+  data.tar.gz: 9d239e735b0333d3662dc663e5fb5dfb83a6bee3
 SHA512:
-  metadata.gz: aa0bd1c5463e94174a4cdf3b8f3a2750ff582deee8dd2fa9c73568393aaa498363bc493a631421092cb92971fe38a97148e448fda1289e0b6bc257e5d458b0d4
-  data.tar.gz: b3d6b231cf413cf101e04eda02da00e2547b9e393584ccc9310b7a427c9ae4cb3a2c3371311b63756656eb6170c842075c11854dc660ce237433eefde65c4914
+  metadata.gz: 9c89ce208dc7b8bcfb95b4bdb8e632b8712cc47bb7348f0f88d67ec83ccf986b69d565b8f66fc880f629ae12f909b13d89beeae960c431e3ab77f1a4f67486c5
+  data.tar.gz: a65992d6071b696bbc25b523002fcb5a32d84d4b0733f07f4a91c52cc4eef96f49947225bdf0431c1443d2a738eb36375dec7b29d2981cb9ee1b95eeea550e40

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.1.4 (September 19, 2016) ##
+
+*   Improve performance when checking for existing records.
+
+    *Scott Ringwelski*
+
 ## 0.1.3 (August 26, 2016) ##
 
 *   Lower the batch size

data/lib/fast_inserter/fast_inserter_base.rb

@@ -94,20 +94,24 @@ module FastInserter
 
     # Queries for the existing values for a given group of values
     def existing_values(group_of_values)
-      values_to_check = ActiveRecord::Base.send(:sanitize_sql_array, ["?", group_of_values])
-      sql = "SELECT #{@variable_column} FROM #{@table_name} WHERE #{existing_values_static_columns} AND #{@variable_column} IN (#{values_to_check})"
+      sql = "SELECT #{@variable_column} FROM #{@table_name} WHERE #{existing_values_static_columns}"
 
       # NOTE: There are more elegant ways to get this field out of the resultset, but each database adaptor returns a different type
       # of result from 'execute(sql)'. Potential classes for 'result' is Array (sqlite), Mysql2::Result (mysql2), PG::Result (pg). Each
       # result can be enumerated into a list of arrays (mysql) or list of hashes (sqlite, pg)
       results = ActiveRecord::Base.connection.execute(sql)
-      results.to_a.map do |result|
+      existing_values = results.to_a.map do |result|
         if result.is_a?(Hash)
           result[@variable_column].to_s
         elsif result.is_a?(Array)
           result[0].to_s
         end
       end
+
+      # Rather than a giant IN query in the sql statement (which can be bad for database performance),
+      # do the filtering of relevant values here in a ruby select.
+      group_of_values_strings = group_of_values.map(&:to_s)
+      existing_values & group_of_values_strings
     end
 
     def existing_values_static_columns

data/lib/fast_inserter/version.rb

@@ -1,3 +1,3 @@
 module FastInserter
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fast_inserter
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Scott Ringwelski
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-08-27 00:00:00.000000000 Z
+date: 2016-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord