bulk_insert 1.7.0 → 1.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 9a76d84f0be15b3ba8e0186439eca425dc9c9d07
-  data.tar.gz: c9fd5eb614204c838fed55e0ee7bc8075dec0b47
+SHA256:
+  metadata.gz: 7f92126ff36c331b40bfc5ca7d261af01410963a5c9dab65824a64f36405e102
+  data.tar.gz: ef41beaeb0f13ff251818e0a57c0e729f2a3bf520d536a24182c2213e0109262
 SHA512:
-  metadata.gz: 533c701360f76f710a875c40f1a0d6c5d463eb8a64c8b32a9144559e8972b85dbceeedda64fd5b7d74587ebb7bfef426217a6c026aab3c1fe408fe7d96e826b8
-  data.tar.gz: 00b3dd6b1c0826ca6c3eec7a64d61f609414be09d266c183e212c6dcc9886c34f731cd7a8d8785e592b6a98eee3933bf25c62c0388d387992308917923ccde61
+  metadata.gz: e6f50b3cd6d20bc348c0cf6c94c3e920a54e26c5b1ba2019209b2521febdffcc4fdf9a3bf39d688990a0e3907fbbf244d3ed53e3ecbe53db283ea3aec7342dbf
+  data.tar.gz: 1849fab8452c635004419e67e8748428e142875aa0c4b4920d4daecab66ed1a2b6ca863124704ae4d847e5c4fce5d38ec8911cb95cc5aad0f533682bc869bc65

data/README.md

@@ -104,7 +104,6 @@ empty the batch so that you can add more rows to it if you want. Note
 that all records saved together will have the same created_at/updated_at
 timestamp (unless one was explicitly set).
 
-
 ### Batch Set Size
 
 By default, the size of the insert is limited to 500 rows at a time.
@@ -149,22 +148,29 @@ Book.bulk_insert(*destination_columns, ignore: true) do |worker|
 end
 ```
 
-### Update Duplicates (MySQL)
+### Update Duplicates (MySQL, PostgreSQL)
 
 If you don't want to ignore duplicate rows but instead want to update them
 then you can use the _update_duplicates_ option. Set this option to true
-and when a duplicate row is found the row will be updated with your new
-values. Default value for this option is false.
+(MySQL) or list unique column names (PostgreSQL) and when a duplicate row
+is found the row will be updated with your new values.
+Default value for this option is false.
 
 ```ruby
 destination_columns = [:title, :author]
 
-# Update duplicate rows
+# Update duplicate rows (MySQL)
 Book.bulk_insert(*destination_columns, update_duplicates: true) do |worker|
   worker.add(...)
   worker.add(...)
   # ...
 end
+
+# Update duplicate rows (PostgreSQL)
+Book.bulk_insert(*destination_columns, update_duplicates: %w[title]) do |worker|
+  worker.add(...)
+  # ...
+end
 ```
 
 ### Return Primary Keys (PostgreSQL, PostGIS)
@@ -185,7 +191,6 @@ end
 worker.result_sets
 ```
 
-
 ## License
 
 BulkInsert is released under the MIT license (see MIT-LICENSE) by

data/lib/bulk_insert/statement_adapters.rb

@@ -0,0 +1,22 @@
+require_relative 'statement_adapters/generic_adapter'
+require_relative 'statement_adapters/mysql_adapter'
+require_relative 'statement_adapters/postgresql_adapter'
+require_relative 'statement_adapters/sqlite_adapter'
+
+module BulkInsert
+  module StatementAdapters
+    def adapter_for(connection)
+      case connection.adapter_name
+      when /^mysql/i
+        MySQLAdapter.new
+      when /\APost(?:greSQL|GIS)/i
+        PostgreSQLAdapter.new
+      when /\ASQLite/i
+        SQLiteAdapter.new
+      else
+        GenericAdapter.new
+      end
+    end
+    module_function :adapter_for
+  end
+end

data/lib/bulk_insert/statement_adapters/base_adapter.rb

@@ -0,0 +1,21 @@
+module BulkInsert
+  module StatementAdapters
+    class BaseAdapter
+      def initialize
+        raise "You cannot initialize base adapter" if self.class == BaseAdapter
+      end
+
+      def insert_ignore_statement
+        raise "Not implemented"
+      end
+
+      def on_conflict_statement(_columns, _ignore, _update_duplicates)
+        raise "Not implemented"
+      end
+
+      def primary_key_return_statement(_primary_key)
+        raise "Not implemented"
+      end
+    end
+  end
+end

data/lib/bulk_insert/statement_adapters/generic_adapter.rb

@@ -0,0 +1,19 @@
+require_relative 'base_adapter'
+
+module BulkInsert
+  module StatementAdapters
+    class GenericAdapter < BaseAdapter
+      def insert_ignore_statement
+        ''
+      end
+
+      def on_conflict_statement(_columns, _ignore, _update_duplicates)
+        ''
+      end
+
+      def primary_key_return_statement(_primary_key)
+        ''
+      end
+    end
+  end
+end

data/lib/bulk_insert/statement_adapters/mysql_adapter.rb

@@ -0,0 +1,24 @@
+require_relative 'base_adapter'
+
+module BulkInsert
+  module StatementAdapters
+    class MySQLAdapter < BaseAdapter
+      def insert_ignore_statement
+        'IGNORE'
+      end
+
+      def on_conflict_statement(columns, _ignore, update_duplicates)
+        return '' unless update_duplicates
+
+        update_values = columns.map do |column|
+          "`#{column.name}`=VALUES(`#{column.name}`)"
+        end.join(', ')
+        ' ON DUPLICATE KEY UPDATE ' + update_values
+      end
+
+      def primary_key_return_statement(_primary_key)
+        ''
+      end
+    end
+  end
+end

data/lib/bulk_insert/statement_adapters/postgresql_adapter.rb

@@ -0,0 +1,26 @@
+require_relative 'base_adapter'
+
+module BulkInsert
+  module StatementAdapters
+    class PostgreSQLAdapter < BaseAdapter
+      def insert_ignore_statement
+        ''
+      end
+
+      def on_conflict_statement(columns, ignore, update_duplicates)
+        if ignore
+          ' ON CONFLICT DO NOTHING'
+        elsif update_duplicates
+          update_values = columns.map do |column|
+            "#{column.name}=EXCLUDED.#{column.name}"
+          end.join(', ')
+          ' ON CONFLICT(' + update_duplicates.join(', ') + ') DO UPDATE SET ' + update_values
+        end
+      end
+
+      def primary_key_return_statement(primary_key)
+        " RETURNING #{primary_key}"
+      end
+    end
+  end
+end

data/lib/bulk_insert/statement_adapters/sqlite_adapter.rb

@@ -0,0 +1,19 @@
+require_relative 'base_adapter'
+
+module BulkInsert
+  module StatementAdapters
+    class SQLiteAdapter < BaseAdapter
+      def insert_ignore_statement
+        'OR IGNORE'
+      end
+
+      def on_conflict_statement(_columns, _ignore, _update_duplicates)
+        ''
+      end
+
+      def primary_key_return_statement(_primary_key)
+        ''
+      end
+    end
+  end
+end

data/lib/bulk_insert/version.rb

@@ -1,6 +1,6 @@
 module BulkInsert
   MAJOR = 1
-  MINOR = 7
+  MINOR = 8
   TINY  = 0
 
   VERSION = [MAJOR, MINOR, TINY].join(".")

data/lib/bulk_insert/worker.rb

@@ -1,3 +1,5 @@
+require_relative 'statement_adapters'
+
 module BulkInsert
   class Worker
     attr_reader :connection
@@ -8,6 +10,8 @@ module BulkInsert
     attr_reader :ignore, :update_duplicates, :result_sets
 
     def initialize(connection, table_name, primary_key, column_names, set_size=500, ignore=false, update_duplicates=false, return_primary_keys=false)
+      @statement_adapter = StatementAdapters.adapter_for(connection)
+
       @connection = connection
       @set_size = set_size
 
@@ -116,8 +120,8 @@ module BulkInsert
 
       if !rows.empty?
         sql << rows.join(",")
-        sql << on_conflict_statement
-        sql << primary_key_return_statement
+        sql << @statement_adapter.on_conflict_statement(@columns, ignore, update_duplicates)
+        sql << @statement_adapter.primary_key_return_statement(@primary_key) if @return_primary_keys
         sql
       else
         false
@@ -125,41 +129,8 @@ module BulkInsert
     end
 
     def insert_sql_statement
+      insert_ignore = @ignore ? @statement_adapter.insert_ignore_statement : ''
       "INSERT #{insert_ignore} INTO #{@table_name} (#{@column_names}) VALUES "
     end
-
-    def insert_ignore
-      if ignore
-        case adapter_name
-        when /^mysql/i
-          'IGNORE'
-        when /\ASQLite/i # SQLite
-          'OR IGNORE'
-        else
-          '' # Not supported
-        end
-      end
-    end
-
-    def primary_key_return_statement
-      if @return_primary_keys && adapter_name =~ /\APost(?:greSQL|GIS)/i
-        " RETURNING #{@primary_key}"
-      else
-        ''
-      end
-    end
-
-    def on_conflict_statement
-      if (adapter_name =~ /\APost(?:greSQL|GIS)/i && ignore )
-        ' ON CONFLICT DO NOTHING'
-      elsif adapter_name =~ /^mysql/i && update_duplicates
-        update_values = @columns.map do |column|
-          "`#{column.name}`=VALUES(`#{column.name}`)"
-        end.join(', ')
-        ' ON DUPLICATE KEY UPDATE ' + update_values
-      else
-        ''
-      end
-    end
   end
 end

data/test/bulk_insert/worker_test.rb

@@ -1,3 +1,4 @@
+require 'minitest/mock'
 require 'test_helper'
 
 class BulkInsertWorkerTest < ActiveSupport::TestCase
@@ -270,93 +271,124 @@ class BulkInsertWorkerTest < ActiveSupport::TestCase
   end
 
   test "adapter dependent mysql methods" do
-    mysql_worker = BulkInsert::Worker.new(
-      Testing.connection,
-      Testing.table_name,
-      'id',
-      %w(greeting age happy created_at updated_at color),
-      500, # batch size
-      true) # ignore
-    mysql_worker.adapter_name = 'MySQL'
-
-    assert_equal mysql_worker.adapter_name, 'MySQL'
-    assert_equal (mysql_worker.adapter_name == 'MySQL'), true
-    assert_equal mysql_worker.ignore, true
-    assert_equal ((mysql_worker.adapter_name == 'MySQL') & mysql_worker.ignore), true
-
-    mysql_worker.add ["Yo", 15, false, nil, nil]
-
-    assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')"
+    connection = Testing.connection
+    connection.stub :adapter_name, 'MySQL' do
+      mysql_worker = BulkInsert::Worker.new(
+        connection,
+        Testing.table_name,
+        'id',
+        %w(greeting age happy created_at updated_at color),
+        500, # batch size
+        true  # ignore
+      )
+
+      assert_equal mysql_worker.adapter_name, 'MySQL'
+      assert_equal (mysql_worker.adapter_name == 'MySQL'), true
+      assert_equal mysql_worker.ignore, true
+      assert_equal ((mysql_worker.adapter_name == 'MySQL') & mysql_worker.ignore), true
+
+      mysql_worker.add ["Yo", 15, false, nil, nil]
+
+      assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')"
+    end
   end
 
   test "adapter dependent mysql methods work for mysql2" do
-    mysql_worker = BulkInsert::Worker.new(
-      Testing.connection,
-      Testing.table_name,
-      'id',
-      %w(greeting age happy created_at updated_at color),
-      500, # batch size
-      true, # ignore
-      true) # update_duplicates
-    mysql_worker.adapter_name = 'Mysql2'
-
-    assert_equal mysql_worker.adapter_name, 'Mysql2'
-    assert mysql_worker.ignore
-
-    mysql_worker.add ["Yo", 15, false, nil, nil]
-
-    assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)"
+    connection = Testing.connection
+    connection.stub :adapter_name, 'Mysql2' do
+      mysql_worker = BulkInsert::Worker.new(
+        connection,
+        Testing.table_name,
+        'id',
+        %w(greeting age happy created_at updated_at color),
+        500, # batch size
+        true, # ignore
+        true) # update_duplicates
+
+      assert_equal mysql_worker.adapter_name, 'Mysql2'
+      assert mysql_worker.ignore
+
+      mysql_worker.add ["Yo", 15, false, nil, nil]
+
+      assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)"
+    end
   end
 
   test "adapter dependent Mysql2Spatial methods" do
-    mysql_worker = BulkInsert::Worker.new(
-      Testing.connection,
-      Testing.table_name,
-      'id',
-      %w(greeting age happy created_at updated_at color),
-      500, # batch size
-      true) # ignore
-    mysql_worker.adapter_name = 'Mysql2Spatial'
+    connection = Testing.connection
+    connection.stub :adapter_name, 'Mysql2Spatial' do
+      mysql_worker = BulkInsert::Worker.new(
+        connection,
+        Testing.table_name,
+        'id',
+        %w(greeting age happy created_at updated_at color),
+        500, # batch size
+        true) # ignore
 
-    assert_equal mysql_worker.adapter_name, 'Mysql2Spatial'
+      assert_equal mysql_worker.adapter_name, 'Mysql2Spatial'
 
-    mysql_worker.add ["Yo", 15, false, nil, nil]
+      mysql_worker.add ["Yo", 15, false, nil, nil]
 
-    assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')"
+      assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')"
+      end
   end
 
   test "adapter dependent postgresql methods" do
-    pgsql_worker = BulkInsert::Worker.new(
-      Testing.connection,
-      Testing.table_name,
-      'id',
-      %w(greeting age happy created_at updated_at color),
-      500, # batch size
-      true, # ignore
-      false, # update duplicates
-      true # return primary keys
-    )
-    pgsql_worker.adapter_name = 'PostgreSQL'
-    pgsql_worker.add ["Yo", 15, false, nil, nil]
+    connection = Testing.connection
+    connection.stub :adapter_name, 'PostgreSQL' do
+      pgsql_worker = BulkInsert::Worker.new(
+        connection,
+        Testing.table_name,
+        'id',
+        %w(greeting age happy created_at updated_at color),
+        500, # batch size
+        true, # ignore
+        false, # update duplicates
+        true # return primary keys
+      )
+
+      pgsql_worker.add ["Yo", 15, false, nil, nil]
+
+      assert_equal pgsql_worker.compose_insert_query, "INSERT  INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id"
+    end
+  end
 
-    assert_equal pgsql_worker.compose_insert_query, "INSERT  INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id"
+  test "adapter dependent postgresql methods (with update_duplicates)" do
+    connection = Testing.connection
+    connection.stub :adapter_name, 'PostgreSQL' do
+      pgsql_worker = BulkInsert::Worker.new(
+        connection,
+        Testing.table_name,
+        'id',
+        %w(greeting age happy created_at updated_at color),
+        500, # batch size
+        false, # ignore
+        %w(greeting age happy), # update duplicates
+        true # return primary keys
+      )
+      pgsql_worker.add ["Yo", 15, false, nil, nil]
+
+      assert_equal pgsql_worker.compose_insert_query, "INSERT  INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON CONFLICT(greeting, age, happy) DO UPDATE SET greeting=EXCLUDED.greeting, age=EXCLUDED.age, happy=EXCLUDED.happy, created_at=EXCLUDED.created_at, updated_at=EXCLUDED.updated_at, color=EXCLUDED.color RETURNING id"
+    end
   end
 
   test "adapter dependent PostGIS methods" do
-    pgsql_worker = BulkInsert::Worker.new(
-      Testing.connection,
-      Testing.table_name,
-      'id',
-      %w(greeting age happy created_at updated_at color),
-      500, # batch size
-      true, # ignore
-      false, # update duplicates
-      true # return primary keys
-    )
-    pgsql_worker.adapter_name = 'PostGIS'
-    pgsql_worker.add ["Yo", 15, false, nil, nil]
-
-    assert_equal pgsql_worker.compose_insert_query, "INSERT  INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id"
+    connection = Testing.connection
+    connection.stub :adapter_name, 'PostGIS' do
+      pgsql_worker = BulkInsert::Worker.new(
+        connection,
+        Testing.table_name,
+        'id',
+        %w(greeting age happy created_at updated_at color),
+        500, # batch size
+        true, # ignore
+        false, # update duplicates
+        true # return primary keys
+      )
+      pgsql_worker.add ["Yo", 15, false, nil, nil]
+
+      assert_equal pgsql_worker.compose_insert_query, "INSERT  INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id"
+    end
   end
 
   test "adapter dependent sqlite3 methods (with lowercase adapter name)" do
@@ -388,17 +420,20 @@ class BulkInsertWorkerTest < ActiveSupport::TestCase
   end
 
   test "mysql adapter can update duplicates" do
-    mysql_worker = BulkInsert::Worker.new(
-      Testing.connection,
-      Testing.table_name,
-      'id',
-      %w(greeting age happy created_at updated_at color),
-      500, # batch size
-      false, # ignore
-      true) # update_duplicates
-    mysql_worker.adapter_name = 'MySQL'
-    mysql_worker.add ["Yo", 15, false, nil, nil]
-
-    assert_equal mysql_worker.compose_insert_query, "INSERT  INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)"
+    connection = Testing.connection
+    connection.stub :adapter_name, 'MySQL' do
+      mysql_worker = BulkInsert::Worker.new(
+        connection,
+        Testing.table_name,
+        'id',
+        %w(greeting age happy created_at updated_at color),
+        500, # batch size
+        false, # ignore
+        true # update_duplicates
+      )
+      mysql_worker.add ["Yo", 15, false, nil, nil]
+
+      assert_equal mysql_worker.compose_insert_query, "INSERT  INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)"
+    end
   end
 end