activerecord-import 0.17.2 → 1.1.0

data/test/support/sqlite3/import_examples.rb

@@ -1,23 +1,12 @@
 # encoding: UTF-8
 def should_support_sqlite3_import_functionality
-  should_support_on_duplicate_key_ignore
+  if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
+    should_support_sqlite_upsert_functionality
+  end
 
   describe "#supports_imports?" do
-    context "and SQLite is 3.7.11 or higher" do
-      it "supports import" do
-        version = ActiveRecord::ConnectionAdapters::SQLite3Adapter::Version.new("3.7.11")
-        assert ActiveRecord::Base.supports_import?(version)
-
-        version = ActiveRecord::ConnectionAdapters::SQLite3Adapter::Version.new("3.7.12")
-        assert ActiveRecord::Base.supports_import?(version)
-      end
-    end
-
-    context "and SQLite less than 3.7.11" do
-      it "doesn't support import" do
-        version = ActiveRecord::ConnectionAdapters::SQLite3Adapter::Version.new("3.7.10")
-        assert !ActiveRecord::Base.supports_import?(version)
-      end
+    it "should support import" do
+      assert ActiveRecord::Base.supports_import?
     end
   end
 
@@ -49,18 +38,193 @@ def should_support_sqlite3_import_functionality
         assert_equal 2500, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
       end
     end
+  end
+end
+
+def should_support_sqlite_upsert_functionality
+  should_support_basic_on_duplicate_key_update
+  should_support_on_duplicate_key_ignore
+
+  describe "#import" do
+    extend ActiveSupport::TestCase::ImportAssertions
+
+    macro(:perform_import) { raise "supply your own #perform_import in a context below" }
+    macro(:updated_topic) { Topic.find(@topic.id) }
+
+    context "with :on_duplicate_key_ignore and validation checks turned off" do
+      let(:columns) { %w( id title author_name author_email_address parent_id ) }
+      let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] }
+      let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] }
+
+      setup do
+        Topic.import columns, values, validate: false
+      end
+
+      it "should not update any records" do
+        result = Topic.import columns, updated_values, on_duplicate_key_ignore: true, validate: false
+        assert_equal [], result.ids
+      end
+    end
+
+    context "with :on_duplicate_key_update and validation checks turned off" do
+      asssertion_group(:should_support_on_duplicate_key_update) do
+        should_not_update_fields_not_mentioned
+        should_update_foreign_keys
+        should_not_update_created_at_on_timestamp_columns
+        should_update_updated_at_on_timestamp_columns
+      end
+
+      context "using a hash" do
+        context "with :columns a hash" do
+          let(:columns) { %w( id title author_name author_email_address parent_id ) }
+          let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] }
+          let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] }
+
+          macro(:perform_import) do |*opts|
+            Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id, columns: update_columns }, validate: false)
+          end
+
+          setup do
+            Topic.import columns, values, validate: false
+            @topic = Topic.find 99
+          end
+
+          it "should not modify the passed in :on_duplicate_key_update columns array" do
+            assert_nothing_raised do
+              columns = %w(title author_name).freeze
+              Topic.import columns, [%w(foo, bar)], on_duplicate_key_update: { columns: columns }
+            end
+          end
+
+          context "using string hash map" do
+            let(:update_columns) { { "title" => "title", "author_email_address" => "author_email_address", "parent_id" => "parent_id" } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned
+          end
+
+          context "using string hash map, but specifying column mismatches" do
+            let(:update_columns) { { "title" => "author_email_address", "author_email_address" => "title", "parent_id" => "parent_id" } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned_with_hash_mappings
+          end
+
+          context "using symbol hash map" do
+            let(:update_columns) { { title: :title, author_email_address: :author_email_address, parent_id: :parent_id } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned
+          end
+
+          context "using symbol hash map, but specifying column mismatches" do
+            let(:update_columns) { { title: :author_email_address, author_email_address: :title, parent_id: :parent_id } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned_with_hash_mappings
+          end
+        end
+
+        context 'with :index_predicate' do
+          let(:columns) { %w( id device_id alarm_type status metadata ) }
+          let(:values) { [[99, 17, 1, 1, 'foo']] }
+          let(:updated_values) { [[99, 17, 1, 2, 'bar']] }
+
+          macro(:perform_import) do |*opts|
+            Alarm.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: [:device_id, :alarm_type], index_predicate: 'status <> 0', columns: [:status] }, validate: false)
+          end
+
+          macro(:updated_alarm) { Alarm.find(@alarm.id) }
+
+          setup do
+            Alarm.import columns, values, validate: false
+            @alarm = Alarm.find 99
+          end
+
+          context 'supports on duplicate key update for partial indexes' do
+            it 'should not update created_at timestamp columns' do
+              Timecop.freeze Chronic.parse("5 minutes from now") do
+                perform_import
+                assert_in_delta @alarm.created_at.to_i, updated_alarm.created_at.to_i, 1
+              end
+            end
+
+            it 'should update updated_at timestamp columns' do
+              time = Chronic.parse("5 minutes from now")
+              Timecop.freeze time do
+                perform_import
+                assert_in_delta time.to_i, updated_alarm.updated_at.to_i, 1
+              end
+            end
+
+            it 'should not update fields not mentioned' do
+              perform_import
+              assert_equal 'foo', updated_alarm.metadata
+            end
+
+            it 'should update fields mentioned with hash mappings' do
+              perform_import
+              assert_equal 2, updated_alarm.status
+            end
+          end
+        end
+
+        context 'with :condition' do
+          let(:columns) { %w( id device_id alarm_type status metadata) }
+          let(:values) { [[99, 17, 1, 1, 'foo']] }
+          let(:updated_values) { [[99, 17, 1, 1, 'bar']] }
+
+          macro(:perform_import) do |*opts|
+            Alarm.import(
+              columns,
+              updated_values,
+              opts.extract_options!.merge(
+                on_duplicate_key_update: {
+                  conflict_target: [:id],
+                  condition: "alarms.metadata NOT LIKE '%foo%'",
+                  columns: [:metadata]
+                },
+                validate: false
+              )
+            )
+          end
+
+          macro(:updated_alarm) { Alarm.find(@alarm.id) }
+
+          setup do
+            Alarm.import columns, values, validate: false
+            @alarm = Alarm.find 99
+          end
+
+          it 'should not update fields not matched' do
+            perform_import
+            assert_equal 'foo', updated_alarm.metadata
+          end
+        end
+
+        context "with no :conflict_target" do
+          context "with no primary key" do
+            it "raises ArgumentError" do
+              error = assert_raises ArgumentError do
+                Rule.import Build(3, :rules), on_duplicate_key_update: [:condition_text], validate: false
+              end
+              assert_match(/Expected :conflict_target to be specified/, error.message)
+            end
+          end
+        end
+
+        context "with no :columns" do
+          let(:columns) { %w( id title author_name author_email_address ) }
+          let(:values) { [[100, "Book", "John Doe", "john@doe.com"]] }
+          let(:updated_values) { [[100, "Title Should Not Change", "Author Should Not Change", "john@nogo.com"]] }
 
-    context "with :on_duplicate_key_update" do
-      let(:topics) { Build(1, :topics) }
+          macro(:perform_import) do |*opts|
+            Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id }, validate: false)
+          end
 
-      it "should log a warning message" do
-        log = StringIO.new
-        logger = Logger.new(log)
-        logger.level = Logger::WARN
-        ActiveRecord::Base.connection.stubs(:logger).returns(logger)
+          setup do
+            Topic.import columns, values, validate: false
+            @topic = Topic.find 100
+          end
 
-        Topic.import topics, on_duplicate_key_update: true
-        assert_match(/Ignoring on_duplicate_key_update/, log.string)
+          should_update_updated_at_on_timestamp_columns
+        end
       end
     end
   end

data/test/synchronize_test.rb

@@ -30,4 +30,12 @@ describe ".synchronize" do
     assert_equal false, topics[1].changed?, "the second record was dirty"
     assert_equal false, topics[2].changed?, "the third record was dirty"
   end
+
+  it "ignores default scope" do
+    # update records outside of ActiveRecord knowing about it
+    Topic.connection.execute( "UPDATE #{Topic.table_name} SET approved='0' WHERE id=#{topics[0].id}", "Updating record 1 without ActiveRecord" )
+
+    Topic.synchronize topics
+    assert_equal false, topics[0].approved
+  end
 end

data/test/test_helper.rb

@@ -26,7 +26,20 @@ end
 
 require 'timecop'
 require 'chronic'
-require 'composite_primary_keys'
+
+begin
+  require 'composite_primary_keys'
+rescue LoadError
+  ENV["SKIP_COMPOSITE_PK"] = "true"
+end
+
+# Support MySQL 5.7
+if ActiveSupport::VERSION::STRING < "4.1"
+  require "active_record/connection_adapters/mysql2_adapter"
+  class ActiveRecord::ConnectionAdapters::Mysql2Adapter
+    NATIVE_DATABASE_TYPES[:primary_key] = "int(11) auto_increment PRIMARY KEY"
+  end
+end
 
 require "ruby-debug" if RUBY_VERSION.to_f < 1.9
 
@@ -35,7 +48,15 @@ adapter = ENV["ARE_DB"] || "sqlite3"
 FileUtils.mkdir_p 'log'
 ActiveRecord::Base.logger = Logger.new("log/test.log")
 ActiveRecord::Base.logger.level = Logger::DEBUG
-ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
+
+if ENV['AR_VERSION'].to_f >= 6.0
+  yaml_config = YAML.load_file(test_dir.join("database.yml"))[adapter]
+  config = ActiveRecord::DatabaseConfigurations::HashConfig.new("test", adapter, yaml_config)
+  ActiveRecord::Base.configurations.configurations << config
+else
+  ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
+end
+
 ActiveRecord::Base.default_timezone = :utc
 
 require "activerecord-import"
@@ -45,7 +66,7 @@ ActiveSupport::Notifications.subscribe(/active_record.sql/) do |_, _, _, _, hsh|
   ActiveRecord::Base.logger.info hsh[:sql]
 end
 
-require "factory_girl"
+require "factory_bot"
 Dir[File.dirname(__FILE__) + "/support/**/*.rb"].each { |file| require file }
 
 # Load base/generic schema

data/test/value_sets_bytes_parser_test.rb

@@ -8,6 +8,15 @@ describe ActiveRecord::Import::ValueSetsBytesParser do
     let(:base_sql) { "INSERT INTO atable (a,b,c)" }
     let(:values) { ["(1,2,3)", "(2,3,4)", "(3,4,5)"] }
 
+    context "when the max allowed bytes is 30 and the base SQL is 26 bytes" do
+      it "should raise ActiveRecord::Import::ValueSetTooLargeError" do
+        error = assert_raises ActiveRecord::Import::ValueSetTooLargeError do
+          parser.parse values, reserved_bytes: base_sql.size, max_bytes: 30
+        end
+        assert_match(/33 bytes exceeds the max allowed for an insert \[30\]/, error.message)
+      end
+    end
+
     context "when the max allowed bytes is 33 and the base SQL is 26 bytes" do
       it "should return 3 value sets when given 3 value sets of 7 bytes a piece" do
         value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 33
@@ -54,7 +63,8 @@ describe ActiveRecord::Import::ValueSetsBytesParser do
       values = [
         "('1','2','3')",
         "('4','5','6')",
-        "('7','8','9')"]
+        "('7','8','9')"
+      ]
 
       base_sql_size_in_bytes = 15
       max_bytes = 30
@@ -79,7 +89,8 @@ describe ActiveRecord::Import::ValueSetsBytesParser do
         # each accented e should be 2 bytes, so each entry is 6 bytes instead of 5
         values = [
           "('é')",
-          "('é')"]
+          "('é')"
+        ]
 
         base_sql_size_in_bytes = 15
         max_bytes = 26

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-import
 version: !ruby/object:Gem::Version
-  version: 0.17.2
+  version: 1.1.0
 platform: ruby
 authors:
 - Zach Dennis
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-14 00:00:00.000000000 Z
+date: 2021-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -67,12 +67,16 @@ files:
 - benchmarks/models/test_innodb.rb
 - benchmarks/models/test_memory.rb
 - benchmarks/models/test_myisam.rb
-- benchmarks/schema/mysql_schema.rb
+- benchmarks/schema/mysql2_schema.rb
 - gemfiles/3.2.gemfile
 - gemfiles/4.0.gemfile
 - gemfiles/4.1.gemfile
 - gemfiles/4.2.gemfile
 - gemfiles/5.0.gemfile
+- gemfiles/5.1.gemfile
+- gemfiles/5.2.gemfile
+- gemfiles/6.0.gemfile
+- gemfiles/6.1.gemfile
 - lib/activerecord-import.rb
 - lib/activerecord-import/active_record/adapters/abstract_adapter.rb
 - lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb
@@ -99,6 +103,7 @@ files:
 - test/adapters/jdbcmysql.rb
 - test/adapters/jdbcpostgresql.rb
 - test/adapters/jdbcsqlite3.rb
+- test/adapters/makara_postgis.rb
 - test/adapters/mysql2.rb
 - test/adapters/mysql2_makara.rb
 - test/adapters/mysql2spatial.rb
@@ -113,7 +118,11 @@ files:
 - test/jdbcmysql/import_test.rb
 - test/jdbcpostgresql/import_test.rb
 - test/jdbcsqlite3/import_test.rb
+- test/makara_postgis/import_test.rb
+- test/models/account.rb
 - test/models/alarm.rb
+- test/models/animal.rb
+- test/models/bike_maker.rb
 - test/models/book.rb
 - test/models/car.rb
 - test/models/chapter.rb
@@ -126,6 +135,8 @@ files:
 - test/models/rule.rb
 - test/models/tag.rb
 - test/models/topic.rb
+- test/models/user.rb
+- test/models/user_token.rb
 - test/models/vendor.rb
 - test/models/widget.rb
 - test/mysql2/import_test.rb
@@ -135,9 +146,10 @@ files:
 - test/postgresql/import_test.rb
 - test/schema/generic_schema.rb
 - test/schema/jdbcpostgresql_schema.rb
-- test/schema/mysql_schema.rb
+- test/schema/mysql2_schema.rb
 - test/schema/postgis_schema.rb
 - test/schema/postgresql_schema.rb
+- test/schema/sqlite3_schema.rb
 - test/schema/version.rb
 - test/sqlite3/import_test.rb
 - test/support/active_support/test_case_extensions.rb
@@ -155,11 +167,11 @@ files:
 - test/travis/database.yml
 - test/value_sets_bytes_parser_test.rb
 - test/value_sets_records_parser_test.rb
-homepage: http://github.com/zdennis/activerecord-import
+homepage: https://github.com/zdennis/activerecord-import
 licenses:
-- Ruby
+- MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -167,22 +179,22 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.2
+      version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.2
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
 summary: Bulk insert extension for ActiveRecord
 test_files:
 - test/adapters/jdbcmysql.rb
 - test/adapters/jdbcpostgresql.rb
 - test/adapters/jdbcsqlite3.rb
+- test/adapters/makara_postgis.rb
 - test/adapters/mysql2.rb
 - test/adapters/mysql2_makara.rb
 - test/adapters/mysql2spatial.rb
@@ -197,7 +209,11 @@ test_files:
 - test/jdbcmysql/import_test.rb
 - test/jdbcpostgresql/import_test.rb
 - test/jdbcsqlite3/import_test.rb
+- test/makara_postgis/import_test.rb
+- test/models/account.rb
 - test/models/alarm.rb
+- test/models/animal.rb
+- test/models/bike_maker.rb
 - test/models/book.rb
 - test/models/car.rb
 - test/models/chapter.rb
@@ -210,6 +226,8 @@ test_files:
 - test/models/rule.rb
 - test/models/tag.rb
 - test/models/topic.rb
+- test/models/user.rb
+- test/models/user_token.rb
 - test/models/vendor.rb
 - test/models/widget.rb
 - test/mysql2/import_test.rb
@@ -219,9 +237,10 @@ test_files:
 - test/postgresql/import_test.rb
 - test/schema/generic_schema.rb
 - test/schema/jdbcpostgresql_schema.rb
-- test/schema/mysql_schema.rb
+- test/schema/mysql2_schema.rb
 - test/schema/postgis_schema.rb
 - test/schema/postgresql_schema.rb
+- test/schema/sqlite3_schema.rb
 - test/schema/version.rb
 - test/sqlite3/import_test.rb
 - test/support/active_support/test_case_extensions.rb