activerecord-import 0.19.0 → 1.0.0

data/test/support/sqlite3/import_examples.rb

@@ -1,6 +1,8 @@
 # encoding: UTF-8
 def should_support_sqlite3_import_functionality
-  should_support_on_duplicate_key_ignore
+  if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
+    should_support_sqlite_upsert_functionality
+  end
 
   describe "#supports_imports?" do
     context "and SQLite is 3.7.11 or higher" do
@@ -49,18 +51,193 @@ def should_support_sqlite3_import_functionality
         assert_equal 2500, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
       end
     end
+  end
+end
+
+def should_support_sqlite_upsert_functionality
+  should_support_basic_on_duplicate_key_update
+  should_support_on_duplicate_key_ignore
+
+  describe "#import" do
+    extend ActiveSupport::TestCase::ImportAssertions
+
+    macro(:perform_import) { raise "supply your own #perform_import in a context below" }
+    macro(:updated_topic) { Topic.find(@topic.id) }
+
+    context "with :on_duplicate_key_ignore and validation checks turned off" do
+      let(:columns) { %w( id title author_name author_email_address parent_id ) }
+      let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] }
+      let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] }
+
+      setup do
+        Topic.import columns, values, validate: false
+      end
+
+      it "should not update any records" do
+        result = Topic.import columns, updated_values, on_duplicate_key_ignore: true, validate: false
+        assert_equal [], result.ids
+      end
+    end
+
+    context "with :on_duplicate_key_update and validation checks turned off" do
+      asssertion_group(:should_support_on_duplicate_key_update) do
+        should_not_update_fields_not_mentioned
+        should_update_foreign_keys
+        should_not_update_created_at_on_timestamp_columns
+        should_update_updated_at_on_timestamp_columns
+      end
+
+      context "using a hash" do
+        context "with :columns a hash" do
+          let(:columns) { %w( id title author_name author_email_address parent_id ) }
+          let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] }
+          let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] }
+
+          macro(:perform_import) do |*opts|
+            Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id, columns: update_columns }, validate: false)
+          end
+
+          setup do
+            Topic.import columns, values, validate: false
+            @topic = Topic.find 99
+          end
+
+          it "should not modify the passed in :on_duplicate_key_update columns array" do
+            assert_nothing_raised do
+              columns = %w(title author_name).freeze
+              Topic.import columns, [%w(foo, bar)], on_duplicate_key_update: { columns: columns }
+            end
+          end
+
+          context "using string hash map" do
+            let(:update_columns) { { "title" => "title", "author_email_address" => "author_email_address", "parent_id" => "parent_id" } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned
+          end
+
+          context "using string hash map, but specifying column mismatches" do
+            let(:update_columns) { { "title" => "author_email_address", "author_email_address" => "title", "parent_id" => "parent_id" } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned_with_hash_mappings
+          end
+
+          context "using symbol hash map" do
+            let(:update_columns) { { title: :title, author_email_address: :author_email_address, parent_id: :parent_id } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned
+          end
+
+          context "using symbol hash map, but specifying column mismatches" do
+            let(:update_columns) { { title: :author_email_address, author_email_address: :title, parent_id: :parent_id } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned_with_hash_mappings
+          end
+        end
+
+        context 'with :index_predicate' do
+          let(:columns) { %w( id device_id alarm_type status metadata ) }
+          let(:values) { [[99, 17, 1, 1, 'foo']] }
+          let(:updated_values) { [[99, 17, 1, 2, 'bar']] }
+
+          macro(:perform_import) do |*opts|
+            Alarm.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: [:device_id, :alarm_type], index_predicate: 'status <> 0', columns: [:status] }, validate: false)
+          end
+
+          macro(:updated_alarm) { Alarm.find(@alarm.id) }
+
+          setup do
+            Alarm.import columns, values, validate: false
+            @alarm = Alarm.find 99
+          end
+
+          context 'supports on duplicate key update for partial indexes' do
+            it 'should not update created_at timestamp columns' do
+              Timecop.freeze Chronic.parse("5 minutes from now") do
+                perform_import
+                assert_in_delta @alarm.created_at.to_i, updated_alarm.created_at.to_i, 1
+              end
+            end
+
+            it 'should update updated_at timestamp columns' do
+              time = Chronic.parse("5 minutes from now")
+              Timecop.freeze time do
+                perform_import
+                assert_in_delta time.to_i, updated_alarm.updated_at.to_i, 1
+              end
+            end
+
+            it 'should not update fields not mentioned' do
+              perform_import
+              assert_equal 'foo', updated_alarm.metadata
+            end
+
+            it 'should update fields mentioned with hash mappings' do
+              perform_import
+              assert_equal 2, updated_alarm.status
+            end
+          end
+        end
+
+        context 'with :condition' do
+          let(:columns) { %w( id device_id alarm_type status metadata) }
+          let(:values) { [[99, 17, 1, 1, 'foo']] }
+          let(:updated_values) { [[99, 17, 1, 1, 'bar']] }
+
+          macro(:perform_import) do |*opts|
+            Alarm.import(
+              columns,
+              updated_values,
+              opts.extract_options!.merge(
+                on_duplicate_key_update: {
+                  conflict_target: [:id],
+                  condition: "alarms.metadata NOT LIKE '%foo%'",
+                  columns: [:metadata]
+                },
+                validate: false
+              )
+            )
+          end
+
+          macro(:updated_alarm) { Alarm.find(@alarm.id) }
+
+          setup do
+            Alarm.import columns, values, validate: false
+            @alarm = Alarm.find 99
+          end
+
+          it 'should not update fields not matched' do
+            perform_import
+            assert_equal 'foo', updated_alarm.metadata
+          end
+        end
+
+        context "with no :conflict_target" do
+          context "with no primary key" do
+            it "raises ArgumentError" do
+              error = assert_raises ArgumentError do
+                Rule.import Build(3, :rules), on_duplicate_key_update: [:condition_text], validate: false
+              end
+              assert_match(/Expected :conflict_target to be specified/, error.message)
+            end
+          end
+        end
+
+        context "with no :columns" do
+          let(:columns) { %w( id title author_name author_email_address ) }
+          let(:values) { [[100, "Book", "John Doe", "john@doe.com"]] }
+          let(:updated_values) { [[100, "Title Should Not Change", "Author Should Not Change", "john@nogo.com"]] }
 
-    context "with :on_duplicate_key_update" do
-      let(:topics) { Build(1, :topics) }
+          macro(:perform_import) do |*opts|
+            Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id }, validate: false)
+          end
 
-      it "should log a warning message" do
-        log = StringIO.new
-        logger = Logger.new(log)
-        logger.level = Logger::WARN
-        ActiveRecord::Base.connection.stubs(:logger).returns(logger)
+          setup do
+            Topic.import columns, values, validate: false
+            @topic = Topic.find 100
+          end
 
-        Topic.import topics, on_duplicate_key_update: true
-        assert_match(/Ignoring on_duplicate_key_update/, log.string)
+          should_update_updated_at_on_timestamp_columns
+        end
       end
     end
   end

data/test/synchronize_test.rb

@@ -30,4 +30,12 @@ describe ".synchronize" do
     assert_equal false, topics[1].changed?, "the second record was dirty"
     assert_equal false, topics[2].changed?, "the third record was dirty"
   end
+
+  it "ignores default scope" do
+    # update records outside of ActiveRecord knowing about it
+    Topic.connection.execute( "UPDATE #{Topic.table_name} SET approved='0' WHERE id=#{topics[0].id}", "Updating record 1 without ActiveRecord" )
+
+    Topic.synchronize topics
+    assert_equal false, topics[0].approved
+  end
 end

data/test/test_helper.rb

@@ -33,6 +33,14 @@ rescue LoadError
   ENV["SKIP_COMPOSITE_PK"] = "true"
 end
 
+# Support MySQL 5.7
+if ActiveSupport::VERSION::STRING < "4.1"
+  require "active_record/connection_adapters/mysql2_adapter"
+  class ActiveRecord::ConnectionAdapters::Mysql2Adapter
+    NATIVE_DATABASE_TYPES[:primary_key] = "int(11) auto_increment PRIMARY KEY"
+  end
+end
+
 require "ruby-debug" if RUBY_VERSION.to_f < 1.9
 
 adapter = ENV["ARE_DB"] || "sqlite3"
@@ -50,7 +58,7 @@ ActiveSupport::Notifications.subscribe(/active_record.sql/) do |_, _, _, _, hsh|
   ActiveRecord::Base.logger.info hsh[:sql]
 end
 
-require "factory_girl"
+require "factory_bot"
 Dir[File.dirname(__FILE__) + "/support/**/*.rb"].each { |file| require file }
 
 # Load base/generic schema

data/test/value_sets_bytes_parser_test.rb

@@ -8,6 +8,15 @@ describe ActiveRecord::Import::ValueSetsBytesParser do
     let(:base_sql) { "INSERT INTO atable (a,b,c)" }
     let(:values) { ["(1,2,3)", "(2,3,4)", "(3,4,5)"] }
 
+    context "when the max allowed bytes is 30 and the base SQL is 26 bytes" do
+      it "should raise ActiveRecord::Import::ValueSetTooLargeError" do
+        error = assert_raises ActiveRecord::Import::ValueSetTooLargeError do
+          parser.parse values, reserved_bytes: base_sql.size, max_bytes: 30
+        end
+        assert_match(/33 bytes exceeds the max allowed for an insert \[30\]/, error.message)
+      end
+    end
+
     context "when the max allowed bytes is 33 and the base SQL is 26 bytes" do
       it "should return 3 value sets when given 3 value sets of 7 bytes a piece" do
         value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 33
@@ -54,7 +63,8 @@ describe ActiveRecord::Import::ValueSetsBytesParser do
       values = [
         "('1','2','3')",
         "('4','5','6')",
-        "('7','8','9')"]
+        "('7','8','9')"
+      ]
 
       base_sql_size_in_bytes = 15
       max_bytes = 30
@@ -79,7 +89,8 @@ describe ActiveRecord::Import::ValueSetsBytesParser do
         # each accented e should be 2 bytes, so each entry is 6 bytes instead of 5
         values = [
           "('é')",
-          "('é')"]
+          "('é')"
+        ]
 
         base_sql_size_in_bytes = 15
         max_bytes = 26

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-import
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Zach Dennis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-13 00:00:00.000000000 Z
+date: 2019-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -74,6 +74,7 @@ files:
 - gemfiles/4.2.gemfile
 - gemfiles/5.0.gemfile
 - gemfiles/5.1.gemfile
+- gemfiles/5.2.gemfile
 - lib/activerecord-import.rb
 - lib/activerecord-import/active_record/adapters/abstract_adapter.rb
 - lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb
@@ -100,6 +101,7 @@ files:
 - test/adapters/jdbcmysql.rb
 - test/adapters/jdbcpostgresql.rb
 - test/adapters/jdbcsqlite3.rb
+- test/adapters/makara_postgis.rb
 - test/adapters/mysql2.rb
 - test/adapters/mysql2_makara.rb
 - test/adapters/mysql2spatial.rb
@@ -114,7 +116,10 @@ files:
 - test/jdbcmysql/import_test.rb
 - test/jdbcpostgresql/import_test.rb
 - test/jdbcsqlite3/import_test.rb
+- test/makara_postgis/import_test.rb
+- test/models/account.rb
 - test/models/alarm.rb
+- test/models/bike_maker.rb
 - test/models/book.rb
 - test/models/car.rb
 - test/models/chapter.rb
@@ -127,6 +132,8 @@ files:
 - test/models/rule.rb
 - test/models/tag.rb
 - test/models/topic.rb
+- test/models/user.rb
+- test/models/user_token.rb
 - test/models/vendor.rb
 - test/models/widget.rb
 - test/mysql2/import_test.rb
@@ -136,9 +143,10 @@ files:
 - test/postgresql/import_test.rb
 - test/schema/generic_schema.rb
 - test/schema/jdbcpostgresql_schema.rb
-- test/schema/mysql_schema.rb
+- test/schema/mysql2_schema.rb
 - test/schema/postgis_schema.rb
 - test/schema/postgresql_schema.rb
+- test/schema/sqlite3_schema.rb
 - test/schema/version.rb
 - test/sqlite3/import_test.rb
 - test/support/active_support/test_case_extensions.rb
@@ -176,7 +184,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: Bulk insert extension for ActiveRecord
@@ -184,6 +192,7 @@ test_files:
 - test/adapters/jdbcmysql.rb
 - test/adapters/jdbcpostgresql.rb
 - test/adapters/jdbcsqlite3.rb
+- test/adapters/makara_postgis.rb
 - test/adapters/mysql2.rb
 - test/adapters/mysql2_makara.rb
 - test/adapters/mysql2spatial.rb
@@ -198,7 +207,10 @@ test_files:
 - test/jdbcmysql/import_test.rb
 - test/jdbcpostgresql/import_test.rb
 - test/jdbcsqlite3/import_test.rb
+- test/makara_postgis/import_test.rb
+- test/models/account.rb
 - test/models/alarm.rb
+- test/models/bike_maker.rb
 - test/models/book.rb
 - test/models/car.rb
 - test/models/chapter.rb
@@ -211,6 +223,8 @@ test_files:
 - test/models/rule.rb
 - test/models/tag.rb
 - test/models/topic.rb
+- test/models/user.rb
+- test/models/user_token.rb
 - test/models/vendor.rb
 - test/models/widget.rb
 - test/mysql2/import_test.rb
@@ -220,9 +234,10 @@ test_files:
 - test/postgresql/import_test.rb
 - test/schema/generic_schema.rb
 - test/schema/jdbcpostgresql_schema.rb
-- test/schema/mysql_schema.rb
+- test/schema/mysql2_schema.rb
 - test/schema/postgis_schema.rb
 - test/schema/postgresql_schema.rb
+- test/schema/sqlite3_schema.rb
 - test/schema/version.rb
 - test/sqlite3/import_test.rb
 - test/support/active_support/test_case_extensions.rb

data/test/schema/mysql_schema.rb

@@ -1,16 +0,0 @@
-ActiveRecord::Schema.define do
-  create_table :books, options: 'ENGINE=MyISAM', force: true do |t|
-    t.column :title, :string, null: false
-    t.column :publisher, :string, null: false, default: 'Default Publisher'
-    t.column :author_name, :string, null: false
-    t.column :created_at, :datetime
-    t.column :created_on, :datetime
-    t.column :updated_at, :datetime
-    t.column :updated_on, :datetime
-    t.column :publish_date, :date
-    t.column :topic_id, :integer
-    t.column :for_sale, :boolean, default: true
-    t.column :status, :integer
-  end
-  execute "ALTER TABLE books ADD FULLTEXT( `title`, `publisher`, `author_name` )"
-end