activerecord-import 0.22.0 → 1.4.1

data/test/support/shared_examples/on_duplicate_key_update.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 def should_support_basic_on_duplicate_key_update
   describe "#import" do
     extend ActiveSupport::TestCase::ImportAssertions
@@ -5,7 +7,248 @@ def should_support_basic_on_duplicate_key_update
     macro(:perform_import) { raise "supply your own #perform_import in a context below" }
     macro(:updated_topic) { Topic.find(@topic.id) }
 
+    context "with lock_version upsert" do
+      describe 'optimistic lock' do
+        it 'lock_version upsert after on_duplcate_key_update by model' do
+          users = [
+            User.new(name: 'Salomon'),
+            User.new(name: 'Nathan')
+          ]
+          User.import(users)
+          assert User.count == users.length
+          User.all.each do |user|
+            assert_equal 0, user.lock_version
+          end
+          updated_users = User.all.map do |user|
+            user.name += ' Rothschild'
+            user
+          end
+          User.import(updated_users, on_duplicate_key_update: [:name])
+          assert User.count == updated_users.length
+          User.all.each_with_index do |user, i|
+            assert_equal user.name, users[i].name + ' Rothschild'
+            assert_equal 1, user.lock_version
+          end
+        end
+
+        it 'lock_version upsert after on_duplcate_key_update by array' do
+          users = [
+            User.new(name: 'Salomon'),
+            User.new(name: 'Nathan')
+          ]
+          User.import(users)
+          assert User.count == users.length
+          User.all.each do |user|
+            assert_equal 0, user.lock_version
+          end
+
+          columns = [:id, :name]
+          updated_values = User.all.map do |user|
+            user.name += ' Rothschild'
+            [user.id, user.name]
+          end
+          User.import(columns, updated_values, on_duplicate_key_update: [:name])
+          assert User.count == updated_values.length
+          User.all.each_with_index do |user, i|
+            assert_equal user.name, users[i].name + ' Rothschild'
+            assert_equal 1, user.lock_version
+          end
+        end
+
+        it 'lock_version upsert after on_duplcate_key_update by hash' do
+          users = [
+            User.new(name: 'Salomon'),
+            User.new(name: 'Nathan')
+          ]
+          User.import(users)
+          assert User.count == users.length
+          User.all.each do |user|
+            assert_equal 0, user.lock_version
+          end
+          updated_values = User.all.map do |user|
+            user.name += ' Rothschild'
+            { id: user.id, name: user.name }
+          end
+          User.import(updated_values, on_duplicate_key_update: [:name])
+          assert User.count == updated_values.length
+          User.all.each_with_index do |user, i|
+            assert_equal user.name, users[i].name + ' Rothschild'
+            assert_equal 1, user.lock_version
+          end
+          updated_values2 = User.all.map do |user|
+            user.name += ' jr.'
+            { id: user.id, name: user.name }
+          end
+          User.import(updated_values2, on_duplicate_key_update: [:name])
+          assert User.count == updated_values2.length
+          User.all.each_with_index do |user, i|
+            assert_equal user.name, users[i].name + ' Rothschild jr.'
+            assert_equal 2, user.lock_version
+          end
+        end
+
+        it 'upsert optimistic lock columns other than lock_version by model' do
+          accounts = [
+            Account.new(name: 'Salomon'),
+            Account.new(name: 'Nathan')
+          ]
+          Account.import(accounts)
+          assert Account.count == accounts.length
+          Account.all.each do |user|
+            assert_equal 0, user.lock
+          end
+          updated_accounts = Account.all.map do |user|
+            user.name += ' Rothschild'
+            user
+          end
+          Account.import(updated_accounts, on_duplicate_key_update: [:id, :name])
+          assert Account.count == updated_accounts.length
+          Account.all.each_with_index do |user, i|
+            assert_equal user.name, accounts[i].name + ' Rothschild'
+            assert_equal 1, user.lock
+          end
+        end
+
+        it 'upsert optimistic lock columns other than lock_version by array' do
+          accounts = [
+            Account.new(name: 'Salomon'),
+            Account.new(name: 'Nathan')
+          ]
+          Account.import(accounts)
+          assert Account.count == accounts.length
+          Account.all.each do |user|
+            assert_equal 0, user.lock
+          end
+
+          columns = [:id, :name]
+          updated_values = Account.all.map do |user|
+            user.name += ' Rothschild'
+            [user.id, user.name]
+          end
+          Account.import(columns, updated_values, on_duplicate_key_update: [:name])
+          assert Account.count == updated_values.length
+          Account.all.each_with_index do |user, i|
+            assert_equal user.name, accounts[i].name + ' Rothschild'
+            assert_equal 1, user.lock
+          end
+        end
+
+        it 'upsert optimistic lock columns other than lock_version by hash' do
+          accounts = [
+            Account.new(name: 'Salomon'),
+            Account.new(name: 'Nathan')
+          ]
+          Account.import(accounts)
+          assert Account.count == accounts.length
+          Account.all.each do |user|
+            assert_equal 0, user.lock
+          end
+          updated_values = Account.all.map do |user|
+            user.name += ' Rothschild'
+            { id: user.id, name: user.name }
+          end
+          Account.import(updated_values, on_duplicate_key_update: [:name])
+          assert Account.count == updated_values.length
+          Account.all.each_with_index do |user, i|
+            assert_equal user.name, accounts[i].name + ' Rothschild'
+            assert_equal 1, user.lock
+          end
+        end
+
+        it 'update the lock_version of models separated by namespaces by model' do
+          makers = [
+            Bike::Maker.new(name: 'Yamaha'),
+            Bike::Maker.new(name: 'Honda')
+          ]
+          Bike::Maker.import(makers)
+          assert Bike::Maker.count == makers.length
+          Bike::Maker.all.each do |maker|
+            assert_equal 0, maker.lock_version
+          end
+          updated_makers = Bike::Maker.all.map do |maker|
+            maker.name += ' bikes'
+            maker
+          end
+          Bike::Maker.import(updated_makers, on_duplicate_key_update: [:name])
+          assert Bike::Maker.count == updated_makers.length
+          Bike::Maker.all.each_with_index do |maker, i|
+            assert_equal maker.name, makers[i].name + ' bikes'
+            assert_equal 1, maker.lock_version
+          end
+        end
+        it 'update the lock_version of models separated by namespaces by array' do
+          makers = [
+            Bike::Maker.new(name: 'Yamaha'),
+            Bike::Maker.new(name: 'Honda')
+          ]
+          Bike::Maker.import(makers)
+          assert Bike::Maker.count == makers.length
+          Bike::Maker.all.each do |maker|
+            assert_equal 0, maker.lock_version
+          end
+
+          columns = [:id, :name]
+          updated_values = Bike::Maker.all.map do |maker|
+            maker.name += ' bikes'
+            [maker.id, maker.name]
+          end
+          Bike::Maker.import(columns, updated_values, on_duplicate_key_update: [:name])
+          assert Bike::Maker.count == updated_values.length
+          Bike::Maker.all.each_with_index do |maker, i|
+            assert_equal maker.name, makers[i].name + ' bikes'
+            assert_equal 1, maker.lock_version
+          end
+        end
+
+        it 'update the lock_version of models separated by namespaces by hash' do
+          makers = [
+            Bike::Maker.new(name: 'Yamaha'),
+            Bike::Maker.new(name: 'Honda')
+          ]
+          Bike::Maker.import(makers)
+          assert Bike::Maker.count == makers.length
+          Bike::Maker.all.each do |maker|
+            assert_equal 0, maker.lock_version
+          end
+          updated_values = Bike::Maker.all.map do |maker|
+            maker.name += ' bikes'
+            { id: maker.id, name: maker.name }
+          end
+          Bike::Maker.import(updated_values, on_duplicate_key_update: [:name])
+          assert Bike::Maker.count == updated_values.length
+          Bike::Maker.all.each_with_index do |maker, i|
+            assert_equal maker.name, makers[i].name + ' bikes'
+            assert_equal 1, maker.lock_version
+          end
+        end
+      end
+    end
+
     context "with :on_duplicate_key_update" do
+      describe 'using :all' do
+        let(:columns) { %w( id title author_name author_email_address parent_id ) }
+        let(:updated_values) { [[99, "Book - 2nd Edition", "Jane Doe", "janedoe@example.com", 57]] }
+
+        macro(:perform_import) do |*opts|
+          Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: :all, validate: false)
+        end
+
+        setup do
+          values = [[99, "Book", "John Doe", "john@doe.com", 17, 3]]
+          Topic.import columns + ['replies_count'], values, validate: false
+        end
+
+        it 'updates all specified columns' do
+          perform_import
+          updated_topic = Topic.find(99)
+          assert_equal 'Book - 2nd Edition', updated_topic.title
+          assert_equal 'Jane Doe', updated_topic.author_name
+          assert_equal 'janedoe@example.com', updated_topic.author_email_address
+          assert_equal 57, updated_topic.parent_id
+          assert_equal 3, updated_topic.replies_count
+        end
+      end
+
       describe "argument safety" do
         it "should not modify the passed in :on_duplicate_key_update array" do
           assert_nothing_raised do
@@ -15,6 +258,26 @@ def should_support_basic_on_duplicate_key_update
         end
       end
 
+      context "with timestamps enabled" do
+        let(:time) { Chronic.parse("5 minutes from now") }
+
+        it 'should not overwrite changed updated_at with current timestamp' do
+          topic = Topic.create(author_name: "Jane Doe", title: "Book")
+          timestamp = Time.now.utc
+          topic.updated_at = timestamp
+          Topic.import [topic], on_duplicate_key_update: :all, validate: false
+          assert_equal timestamp.to_s, Topic.last.updated_at.to_s
+        end
+
+        it 'should update updated_at with current timestamp' do
+          topic = Topic.create(author_name: "Jane Doe", title: "Book")
+          Timecop.freeze(time) do
+            Topic.import [topic], on_duplicate_key_update: [:updated_at], validate: false
+            assert_in_delta time.to_i, topic.reload.updated_at.to_i, 1.second
+          end
+        end
+      end
+
       context "with validation checks turned off" do
         asssertion_group(:should_support_on_duplicate_key_update) do
           should_not_update_fields_not_mentioned

data/test/support/shared_examples/recursive_import.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 def should_support_recursive_import
   describe "importing objects with associations" do
     let(:new_topics) { Build(num_topics, :topic_with_book) }
@@ -11,7 +13,7 @@ def should_support_recursive_import
     let(:num_chapters) { 18 }
     let(:num_endnotes) { 24 }
 
-    let(:new_question_with_rule) { FactoryGirl.build :question, :with_rule }
+    let(:new_question_with_rule) { FactoryBot.build :question, :with_rule }
 
     it 'imports top level' do
       assert_difference "Topic.count", +num_topics do
@@ -116,7 +118,10 @@ def should_support_recursive_import
     end
 
     it "imports an imported belongs_to association id" do
-      books = new_topics[0].books.to_a
+      first_new_topic = new_topics[0]
+      second_new_topic = new_topics[1]
+
+      books = first_new_topic.books.to_a
       Topic.import new_topics, validate: false
 
       assert_difference "Book.count", books.size do
@@ -124,7 +129,25 @@ def should_support_recursive_import
       end
 
       books.each do |book|
-        assert_not_nil book.topic_id
+        assert_equal book.topic_id, first_new_topic.id
+      end
+
+      books.each { |book| book.topic_id = second_new_topic.id }
+      assert_no_difference "Book.count", books.size do
+        Book.import books, validate: false, on_duplicate_key_update: [:topic_id]
+      end
+
+      books.each do |book|
+        assert_equal book.topic_id, second_new_topic.id
+      end
+
+      books.each { |book| book.topic_id = nil }
+      assert_no_difference "Book.count", books.size do
+        Book.import books, validate: false, on_duplicate_key_update: [:topic_id]
+      end
+
+      books.each do |book|
+        assert_equal book.topic_id, nil
       end
     end
 
@@ -155,7 +178,7 @@ def should_support_recursive_import
       end
     end
 
-    # If adapter supports on_duplicate_key_update, it is only applied to top level models so that SQL with invalid
+    # If adapter supports on_duplicate_key_update and specific columns are specified, it is only applied to top level models so that SQL with invalid
     # columns, keys, etc isn't generated for child associations when doing recursive import
     if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
       describe "on_duplicate_key_update" do
@@ -169,6 +192,55 @@ def should_support_recursive_import
             end
           end
         end
+
+        context "when :all fields are updated" do
+          setup do
+            Topic.import new_topics, recursive: true
+          end
+
+          it "updates associated objects" do
+            new_author_name = 'Richard Bachman'
+            topic = new_topics.first
+            topic.books.each do |book|
+              book.author_name = new_author_name
+            end
+            assert_nothing_raised do
+              Topic.import new_topics, recursive: true, on_duplicate_key_update: :all
+            end
+            Topic.find(topic.id).books.each do |book|
+              assert_equal new_author_name, book.author_name
+            end
+          end
+        end
+      end
+    end
+
+    # If returning option is provided, it is only applied to top level models so that SQL with invalid
+    # columns, keys, etc isn't generated for child associations when doing recursive import
+    describe "returning" do
+      let(:new_topics) { Build(1, :topic_with_book) }
+
+      it "imports objects with associations" do
+        assert_difference "Topic.count", +1 do
+          Topic.import new_topics, recursive: true, returning: [:content], validate: false
+          new_topics.each do |topic|
+            assert_not_nil topic.id
+          end
+        end
+      end
+    end
+
+    # If no returning option is provided, it is ignored
+    describe "no returning" do
+      let(:new_topics) { Build(1, :topic_with_book) }
+
+      it "is ignored and imports objects with associations" do
+        assert_difference "Topic.count", +1 do
+          Topic.import new_topics, recursive: true, no_returning: true, validate: false
+          new_topics.each do |topic|
+            assert_not_nil topic.id
+          end
+        end
       end
     end
   end

data/test/support/sqlite3/import_examples.rb

@@ -1,23 +1,13 @@
-# encoding: UTF-8
+# frozen_string_literal: true
+
 def should_support_sqlite3_import_functionality
-  should_support_on_duplicate_key_ignore
+  if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
+    should_support_sqlite_upsert_functionality
+  end
 
   describe "#supports_imports?" do
-    context "and SQLite is 3.7.11 or higher" do
-      it "supports import" do
-        version = ActiveRecord::ConnectionAdapters::SQLite3Adapter::Version.new("3.7.11")
-        assert ActiveRecord::Base.supports_import?(version)
-
-        version = ActiveRecord::ConnectionAdapters::SQLite3Adapter::Version.new("3.7.12")
-        assert ActiveRecord::Base.supports_import?(version)
-      end
-    end
-
-    context "and SQLite less than 3.7.11" do
-      it "doesn't support import" do
-        version = ActiveRecord::ConnectionAdapters::SQLite3Adapter::Version.new("3.7.10")
-        assert !ActiveRecord::Base.supports_import?(version)
-      end
+    it "should support import" do
+      assert ActiveRecord::Base.supports_import?
     end
   end
 
@@ -49,18 +39,193 @@ def should_support_sqlite3_import_functionality
         assert_equal 2500, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
       end
     end
+  end
+end
+
+def should_support_sqlite_upsert_functionality
+  should_support_basic_on_duplicate_key_update
+  should_support_on_duplicate_key_ignore
+
+  describe "#import" do
+    extend ActiveSupport::TestCase::ImportAssertions
+
+    macro(:perform_import) { raise "supply your own #perform_import in a context below" }
+    macro(:updated_topic) { Topic.find(@topic.id) }
+
+    context "with :on_duplicate_key_ignore and validation checks turned off" do
+      let(:columns) { %w( id title author_name author_email_address parent_id ) }
+      let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] }
+      let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] }
+
+      setup do
+        Topic.import columns, values, validate: false
+      end
+
+      it "should not update any records" do
+        result = Topic.import columns, updated_values, on_duplicate_key_ignore: true, validate: false
+        assert_equal [], result.ids
+      end
+    end
+
+    context "with :on_duplicate_key_update and validation checks turned off" do
+      asssertion_group(:should_support_on_duplicate_key_update) do
+        should_not_update_fields_not_mentioned
+        should_update_foreign_keys
+        should_not_update_created_at_on_timestamp_columns
+        should_update_updated_at_on_timestamp_columns
+      end
+
+      context "using a hash" do
+        context "with :columns a hash" do
+          let(:columns) { %w( id title author_name author_email_address parent_id ) }
+          let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] }
+          let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] }
+
+          macro(:perform_import) do |*opts|
+            Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id, columns: update_columns }, validate: false)
+          end
+
+          setup do
+            Topic.import columns, values, validate: false
+            @topic = Topic.find 99
+          end
+
+          it "should not modify the passed in :on_duplicate_key_update columns array" do
+            assert_nothing_raised do
+              columns = %w(title author_name).freeze
+              Topic.import columns, [%w(foo, bar)], on_duplicate_key_update: { columns: columns }
+            end
+          end
+
+          context "using string hash map" do
+            let(:update_columns) { { "title" => "title", "author_email_address" => "author_email_address", "parent_id" => "parent_id" } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned
+          end
+
+          context "using string hash map, but specifying column mismatches" do
+            let(:update_columns) { { "title" => "author_email_address", "author_email_address" => "title", "parent_id" => "parent_id" } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned_with_hash_mappings
+          end
+
+          context "using symbol hash map" do
+            let(:update_columns) { { title: :title, author_email_address: :author_email_address, parent_id: :parent_id } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned
+          end
+
+          context "using symbol hash map, but specifying column mismatches" do
+            let(:update_columns) { { title: :author_email_address, author_email_address: :title, parent_id: :parent_id } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned_with_hash_mappings
+          end
+        end
+
+        context 'with :index_predicate' do
+          let(:columns) { %w( id device_id alarm_type status metadata ) }
+          let(:values) { [[99, 17, 1, 1, 'foo']] }
+          let(:updated_values) { [[99, 17, 1, 2, 'bar']] }
+
+          macro(:perform_import) do |*opts|
+            Alarm.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: [:device_id, :alarm_type], index_predicate: 'status <> 0', columns: [:status] }, validate: false)
+          end
+
+          macro(:updated_alarm) { Alarm.find(@alarm.id) }
+
+          setup do
+            Alarm.import columns, values, validate: false
+            @alarm = Alarm.find 99
+          end
+
+          context 'supports on duplicate key update for partial indexes' do
+            it 'should not update created_at timestamp columns' do
+              Timecop.freeze Chronic.parse("5 minutes from now") do
+                perform_import
+                assert_in_delta @alarm.created_at.to_i, updated_alarm.created_at.to_i, 1
+              end
+            end
+
+            it 'should update updated_at timestamp columns' do
+              time = Chronic.parse("5 minutes from now")
+              Timecop.freeze time do
+                perform_import
+                assert_in_delta time.to_i, updated_alarm.updated_at.to_i, 1
+              end
+            end
+
+            it 'should not update fields not mentioned' do
+              perform_import
+              assert_equal 'foo', updated_alarm.metadata
+            end
+
+            it 'should update fields mentioned with hash mappings' do
+              perform_import
+              assert_equal 2, updated_alarm.status
+            end
+          end
+        end
+
+        context 'with :condition' do
+          let(:columns) { %w( id device_id alarm_type status metadata) }
+          let(:values) { [[99, 17, 1, 1, 'foo']] }
+          let(:updated_values) { [[99, 17, 1, 1, 'bar']] }
+
+          macro(:perform_import) do |*opts|
+            Alarm.import(
+              columns,
+              updated_values,
+              opts.extract_options!.merge(
+                on_duplicate_key_update: {
+                  conflict_target: [:id],
+                  condition: "alarms.metadata NOT LIKE '%foo%'",
+                  columns: [:metadata]
+                },
+                validate: false
+              )
+            )
+          end
+
+          macro(:updated_alarm) { Alarm.find(@alarm.id) }
+
+          setup do
+            Alarm.import columns, values, validate: false
+            @alarm = Alarm.find 99
+          end
+
+          it 'should not update fields not matched' do
+            perform_import
+            assert_equal 'foo', updated_alarm.metadata
+          end
+        end
+
+        context "with no :conflict_target" do
+          context "with no primary key" do
+            it "raises ArgumentError" do
+              error = assert_raises ArgumentError do
+                Rule.import Build(3, :rules), on_duplicate_key_update: [:condition_text], validate: false
+              end
+              assert_match(/Expected :conflict_target to be specified/, error.message)
+            end
+          end
+        end
+
+        context "with no :columns" do
+          let(:columns) { %w( id title author_name author_email_address ) }
+          let(:values) { [[100, "Book", "John Doe", "john@doe.com"]] }
+          let(:updated_values) { [[100, "Title Should Not Change", "Author Should Not Change", "john@nogo.com"]] }
 
-    context "with :on_duplicate_key_update" do
-      let(:topics) { Build(1, :topics) }
+          macro(:perform_import) do |*opts|
+            Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id }, validate: false)
+          end
 
-      it "should log a warning message" do
-        log = StringIO.new
-        logger = Logger.new(log)
-        logger.level = Logger::WARN
-        ActiveRecord::Base.connection.stubs(:logger).returns(logger)
+          setup do
+            Topic.import columns, values, validate: false
+            @topic = Topic.find 100
+          end
 
-        Topic.import topics, on_duplicate_key_update: true
-        assert_match(/Ignoring on_duplicate_key_update/, log.string)
+          should_update_updated_at_on_timestamp_columns
+        end
       end
     end
   end

data/test/synchronize_test.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require File.expand_path('../test_helper', __FILE__)
 
 describe ".synchronize" do