activerecord-import 0.10.0 → 1.0.8

data/test/support/sqlite3/import_examples.rb

@@ -0,0 +1,231 @@
+# encoding: UTF-8
+def should_support_sqlite3_import_functionality
+  if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
+    should_support_sqlite_upsert_functionality
+  end
+
+  describe "#supports_imports?" do
+    it "should support import" do
+      assert ActiveRecord::Base.supports_import?
+    end
+  end
+
+  describe "#import" do
+    it "imports with a single insert on SQLite 3.7.11 or higher" do
+      assert_difference "Topic.count", +507 do
+        result = Topic.import Build(7, :topics)
+        assert_equal 1, result.num_inserts, "Failed to issue a single INSERT statement. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+        assert_equal 7, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+
+        result = Topic.import Build(500, :topics)
+        assert_equal 1, result.num_inserts, "Failed to issue a single INSERT statement. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+        assert_equal 507, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+      end
+    end
+
+    it "imports with a two inserts on SQLite 3.7.11 or higher" do
+      assert_difference "Topic.count", +501 do
+        result = Topic.import Build(501, :topics)
+        assert_equal 2, result.num_inserts, "Failed to issue a two INSERT statements. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+        assert_equal 501, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+      end
+    end
+
+    it "imports with a five inserts on SQLite 3.7.11 or higher" do
+      assert_difference "Topic.count", +2500 do
+        result = Topic.import Build(2500, :topics)
+        assert_equal 5, result.num_inserts, "Failed to issue a two INSERT statements. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+        assert_equal 2500, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+      end
+    end
+  end
+end
+
+def should_support_sqlite_upsert_functionality
+  should_support_basic_on_duplicate_key_update
+  should_support_on_duplicate_key_ignore
+
+  describe "#import" do
+    extend ActiveSupport::TestCase::ImportAssertions
+
+    macro(:perform_import) { raise "supply your own #perform_import in a context below" }
+    macro(:updated_topic) { Topic.find(@topic.id) }
+
+    context "with :on_duplicate_key_ignore and validation checks turned off" do
+      let(:columns) { %w( id title author_name author_email_address parent_id ) }
+      let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] }
+      let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] }
+
+      setup do
+        Topic.import columns, values, validate: false
+      end
+
+      it "should not update any records" do
+        result = Topic.import columns, updated_values, on_duplicate_key_ignore: true, validate: false
+        assert_equal [], result.ids
+      end
+    end
+
+    context "with :on_duplicate_key_update and validation checks turned off" do
+      asssertion_group(:should_support_on_duplicate_key_update) do
+        should_not_update_fields_not_mentioned
+        should_update_foreign_keys
+        should_not_update_created_at_on_timestamp_columns
+        should_update_updated_at_on_timestamp_columns
+      end
+
+      context "using a hash" do
+        context "with :columns a hash" do
+          let(:columns) { %w( id title author_name author_email_address parent_id ) }
+          let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] }
+          let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] }
+
+          macro(:perform_import) do |*opts|
+            Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id, columns: update_columns }, validate: false)
+          end
+
+          setup do
+            Topic.import columns, values, validate: false
+            @topic = Topic.find 99
+          end
+
+          it "should not modify the passed in :on_duplicate_key_update columns array" do
+            assert_nothing_raised do
+              columns = %w(title author_name).freeze
+              Topic.import columns, [%w(foo, bar)], on_duplicate_key_update: { columns: columns }
+            end
+          end
+
+          context "using string hash map" do
+            let(:update_columns) { { "title" => "title", "author_email_address" => "author_email_address", "parent_id" => "parent_id" } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned
+          end
+
+          context "using string hash map, but specifying column mismatches" do
+            let(:update_columns) { { "title" => "author_email_address", "author_email_address" => "title", "parent_id" => "parent_id" } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned_with_hash_mappings
+          end
+
+          context "using symbol hash map" do
+            let(:update_columns) { { title: :title, author_email_address: :author_email_address, parent_id: :parent_id } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned
+          end
+
+          context "using symbol hash map, but specifying column mismatches" do
+            let(:update_columns) { { title: :author_email_address, author_email_address: :title, parent_id: :parent_id } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned_with_hash_mappings
+          end
+        end
+
+        context 'with :index_predicate' do
+          let(:columns) { %w( id device_id alarm_type status metadata ) }
+          let(:values) { [[99, 17, 1, 1, 'foo']] }
+          let(:updated_values) { [[99, 17, 1, 2, 'bar']] }
+
+          macro(:perform_import) do |*opts|
+            Alarm.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: [:device_id, :alarm_type], index_predicate: 'status <> 0', columns: [:status] }, validate: false)
+          end
+
+          macro(:updated_alarm) { Alarm.find(@alarm.id) }
+
+          setup do
+            Alarm.import columns, values, validate: false
+            @alarm = Alarm.find 99
+          end
+
+          context 'supports on duplicate key update for partial indexes' do
+            it 'should not update created_at timestamp columns' do
+              Timecop.freeze Chronic.parse("5 minutes from now") do
+                perform_import
+                assert_in_delta @alarm.created_at.to_i, updated_alarm.created_at.to_i, 1
+              end
+            end
+
+            it 'should update updated_at timestamp columns' do
+              time = Chronic.parse("5 minutes from now")
+              Timecop.freeze time do
+                perform_import
+                assert_in_delta time.to_i, updated_alarm.updated_at.to_i, 1
+              end
+            end
+
+            it 'should not update fields not mentioned' do
+              perform_import
+              assert_equal 'foo', updated_alarm.metadata
+            end
+
+            it 'should update fields mentioned with hash mappings' do
+              perform_import
+              assert_equal 2, updated_alarm.status
+            end
+          end
+        end
+
+        context 'with :condition' do
+          let(:columns) { %w( id device_id alarm_type status metadata) }
+          let(:values) { [[99, 17, 1, 1, 'foo']] }
+          let(:updated_values) { [[99, 17, 1, 1, 'bar']] }
+
+          macro(:perform_import) do |*opts|
+            Alarm.import(
+              columns,
+              updated_values,
+              opts.extract_options!.merge(
+                on_duplicate_key_update: {
+                  conflict_target: [:id],
+                  condition: "alarms.metadata NOT LIKE '%foo%'",
+                  columns: [:metadata]
+                },
+                validate: false
+              )
+            )
+          end
+
+          macro(:updated_alarm) { Alarm.find(@alarm.id) }
+
+          setup do
+            Alarm.import columns, values, validate: false
+            @alarm = Alarm.find 99
+          end
+
+          it 'should not update fields not matched' do
+            perform_import
+            assert_equal 'foo', updated_alarm.metadata
+          end
+        end
+
+        context "with no :conflict_target" do
+          context "with no primary key" do
+            it "raises ArgumentError" do
+              error = assert_raises ArgumentError do
+                Rule.import Build(3, :rules), on_duplicate_key_update: [:condition_text], validate: false
+              end
+              assert_match(/Expected :conflict_target to be specified/, error.message)
+            end
+          end
+        end
+
+        context "with no :columns" do
+          let(:columns) { %w( id title author_name author_email_address ) }
+          let(:values) { [[100, "Book", "John Doe", "john@doe.com"]] }
+          let(:updated_values) { [[100, "Title Should Not Change", "Author Should Not Change", "john@nogo.com"]] }
+
+          macro(:perform_import) do |*opts|
+            Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id }, validate: false)
+          end
+
+          setup do
+            Topic.import columns, values, validate: false
+            @topic = Topic.find 100
+          end
+
+          should_update_updated_at_on_timestamp_columns
+        end
+      end
+    end
+  end
+end

data/test/synchronize_test.rb

@@ -1,8 +1,8 @@
 require File.expand_path('../test_helper', __FILE__)
 
 describe ".synchronize" do
-  let(:topics){ Generate(3, :topics) }
-  let(:titles){ %w(one two three) }
+  let(:topics) { Generate(3, :topics) }
+  let(:titles) { %w(one two three) }
 
   setup do
     # update records outside of ActiveRecord knowing about it
@@ -30,4 +30,12 @@ describe ".synchronize" do
     assert_equal false, topics[1].changed?, "the second record was dirty"
     assert_equal false, topics[2].changed?, "the third record was dirty"
   end
+
+  it "ignores default scope" do
+    # update records outside of ActiveRecord knowing about it
+    Topic.connection.execute( "UPDATE #{Topic.table_name} SET approved='0' WHERE id=#{topics[0].id}", "Updating record 1 without ActiveRecord" )
+
+    Topic.synchronize topics
+    assert_equal false, topics[0].approved
+  end
 end

data/test/test_helper.rb

@@ -10,19 +10,37 @@ ENV["RAILS_ENV"] = "test"
 require "bundler"
 Bundler.setup
 
+require 'pry' unless RbConfig::CONFIG["RUBY_INSTALL_NAME"] =~ /jruby/
+
 require "active_record"
 require "active_record/fixtures"
 require "active_support/test_case"
 
-if ActiveSupport::VERSION::STRING < "4.1"
+if ActiveSupport::VERSION::STRING < "4.0"
   require 'test/unit'
+  require 'mocha/test_unit'
 else
   require 'active_support/testing/autorun'
+  require "mocha/mini_test"
 end
 
 require 'timecop'
 require 'chronic'
 
+begin
+  require 'composite_primary_keys'
+rescue LoadError
+  ENV["SKIP_COMPOSITE_PK"] = "true"
+end
+
+# Support MySQL 5.7
+if ActiveSupport::VERSION::STRING < "4.1"
+  require "active_record/connection_adapters/mysql2_adapter"
+  class ActiveRecord::ConnectionAdapters::Mysql2Adapter
+    NATIVE_DATABASE_TYPES[:primary_key] = "int(11) auto_increment PRIMARY KEY"
+  end
+end
+
 require "ruby-debug" if RUBY_VERSION.to_f < 1.9
 
 adapter = ENV["ARE_DB"] || "sqlite3"
@@ -30,26 +48,36 @@ adapter = ENV["ARE_DB"] || "sqlite3"
 FileUtils.mkdir_p 'log'
 ActiveRecord::Base.logger = Logger.new("log/test.log")
 ActiveRecord::Base.logger.level = Logger::DEBUG
-ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
+
+if ENV['AR_VERSION'].to_f >= 6.0
+  yaml_config = YAML.load_file(test_dir.join("database.yml"))[adapter]
+  config = ActiveRecord::DatabaseConfigurations::HashConfig.new("test", adapter, yaml_config)
+  ActiveRecord::Base.configurations.configurations << config
+else
+  ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
+end
+
 ActiveRecord::Base.default_timezone = :utc
 
 require "activerecord-import"
-ActiveRecord::Base.establish_connection "test"
+ActiveRecord::Base.establish_connection :test
 
-ActiveSupport::Notifications.subscribe(/active_record.sql/) do |event, _, _, _, hsh|
+ActiveSupport::Notifications.subscribe(/active_record.sql/) do |_, _, _, _, hsh|
   ActiveRecord::Base.logger.info hsh[:sql]
 end
 
-require "factory_girl"
-Dir[File.dirname(__FILE__) + "/support/**/*.rb"].each{ |file| require file }
+require "factory_bot"
+Dir[File.dirname(__FILE__) + "/support/**/*.rb"].each { |file| require file }
 
 # Load base/generic schema
 require test_dir.join("schema/version")
 require test_dir.join("schema/generic_schema")
 adapter_schema = test_dir.join("schema/#{adapter}_schema.rb")
-require adapter_schema if File.exists?(adapter_schema)
+require adapter_schema if File.exist?(adapter_schema)
 
-Dir[File.dirname(__FILE__) + "/models/*.rb"].each{ |file| require file }
+Dir[File.dirname(__FILE__) + "/models/*.rb"].each { |file| require file }
 
 # Prevent this deprecation warning from breaking the tests.
 Rake::FileList.send(:remove_method, :import)
+
+ActiveSupport::TestCase.test_order = :random if ENV['AR_VERSION'].to_f >= 4.2

data/test/travis/database.yml

@@ -5,31 +5,34 @@ common: &common
   host:     localhost
   database: activerecord_import_test
 
-mysql: &mysql
+jdbcpostgresql: &postgresql
   <<: *common
-  adapter: mysql
+  username: postgres
+  adapter: jdbcpostgresql
+  min_messages: warning
+
+jdbcmysql: &mysql2
+  <<: *common
+  adapter: jdbcmysql
+
+jdbcsqlite3: &sqlite3
+  <<: *common
+  adapter: jdbcsqlite3
 
 mysql2: &mysql2
   <<: *common
   adapter: mysql2
 
-mysqlspatial:
-  <<: *mysql
-
 mysql2spatial:
   <<: *mysql2
 
-em_mysql2:
-  <<: *common
-  adapter: em_mysql2
-  pool: 5
+mysql2_makara:
+  <<: *mysql2
 
-seamless_database_pool:
+oracle:
   <<: *common
-  adapter: seamless_database_pool
-  pool_adapter: mysql2
-  master:
-    host: localhost
+  adapter: oracle
+  min_messages: debug
 
 postgresql: &postgresql
   <<: *common
@@ -37,13 +40,19 @@ postgresql: &postgresql
   adapter: postgresql
   min_messages: warning
 
+postresql_makara:
+  <<: *postgresql
+
 postgis:
   <<: *postgresql
 
-oracle:
+seamless_database_pool:
   <<: *common
-  adapter: oracle
-  min_messages: debug
+  adapter: seamless_database_pool
+  pool_adapter: mysql2
+  prepared_statements: false
+  master:
+    host: localhost
 
 sqlite:
   adapter: sqlite

data/test/value_sets_bytes_parser_test.rb

@@ -4,48 +4,57 @@ require 'activerecord-import/value_sets_parser'
 
 describe ActiveRecord::Import::ValueSetsBytesParser do
   context "#parse - computing insert value sets" do
-    let(:parser){ ActiveRecord::Import::ValueSetsBytesParser }
-    let(:base_sql){ "INSERT INTO atable (a,b,c)" }
-    let(:values){ [ "(1,2,3)", "(2,3,4)", "(3,4,5)" ] }
+    let(:parser) { ActiveRecord::Import::ValueSetsBytesParser }
+    let(:base_sql) { "INSERT INTO atable (a,b,c)" }
+    let(:values) { ["(1,2,3)", "(2,3,4)", "(3,4,5)"] }
+
+    context "when the max allowed bytes is 30 and the base SQL is 26 bytes" do
+      it "should raise ActiveRecord::Import::ValueSetTooLargeError" do
+        error = assert_raises ActiveRecord::Import::ValueSetTooLargeError do
+          parser.parse values, reserved_bytes: base_sql.size, max_bytes: 30
+        end
+        assert_match(/33 bytes exceeds the max allowed for an insert \[30\]/, error.message)
+      end
+    end
 
     context "when the max allowed bytes is 33 and the base SQL is 26 bytes" do
       it "should return 3 value sets when given 3 value sets of 7 bytes a piece" do
-        value_sets = parser.parse values, :reserved_bytes => base_sql.size, :max_bytes => 33
+        value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 33
         assert_equal 3, value_sets.size
       end
     end
 
     context "when the max allowed bytes is 40 and the base SQL is 26 bytes" do
       it "should return 3 value sets when given 3 value sets of 7 bytes a piece" do
-        value_sets = parser.parse values, :reserved_bytes => base_sql.size, :max_bytes => 40
+        value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 40
         assert_equal 3, value_sets.size
       end
     end
 
     context "when the max allowed bytes is 41 and the base SQL is 26 bytes" do
       it "should return 2 value sets when given 2 value sets of 7 bytes a piece" do
-        value_sets = parser.parse values, :reserved_bytes => base_sql.size, :max_bytes => 41
+        value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 41
         assert_equal 2, value_sets.size
       end
     end
 
     context "when the max allowed bytes is 48 and the base SQL is 26 bytes" do
       it "should return 2 value sets when given 2 value sets of 7 bytes a piece" do
-        value_sets = parser.parse values, :reserved_bytes => base_sql.size, :max_bytes => 48
+        value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 48
         assert_equal 2, value_sets.size
       end
     end
 
     context "when the max allowed bytes is 49 and the base SQL is 26 bytes" do
       it "should return 1 value sets when given 1 value sets of 7 bytes a piece" do
-        value_sets = parser.parse values, :reserved_bytes => base_sql.size, :max_bytes => 49
+        value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 49
         assert_equal 1, value_sets.size
       end
     end
 
     context "when the max allowed bytes is 999999 and the base SQL is 26 bytes" do
       it "should return 1 value sets when given 1 value sets of 7 bytes a piece" do
-        value_sets = parser.parse values, :reserved_bytes => base_sql.size, :max_bytes => 999999
+        value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 999_999
         assert_equal 1, value_sets.size
       end
     end
@@ -54,9 +63,9 @@ describe ActiveRecord::Import::ValueSetsBytesParser do
       values = [
         "('1','2','3')",
         "('4','5','6')",
-        "('7','8','9')" ]
+        "('7','8','9')"
+      ]
 
-      values_size_in_bytes = values.sum {|value| value.bytesize }
       base_sql_size_in_bytes = 15
       max_bytes = 30
 
@@ -64,8 +73,9 @@ describe ActiveRecord::Import::ValueSetsBytesParser do
       assert_equal 3, value_sets.size, 'Three value sets were expected!'
 
       # Each element in the value_sets array must be an array
-      value_sets.each_with_index { |e,i|
-        assert_kind_of Array, e, "Element #{i} was expected to be an Array!" }
+      value_sets.each_with_index do |e, i|
+        assert_kind_of Array, e, "Element #{i} was expected to be an Array!"
+      end
 
       # Each element in the values array should have a 1:1 correlation to the elements
       # in the returned value_sets arrays
@@ -79,18 +89,16 @@ describe ActiveRecord::Import::ValueSetsBytesParser do
         # each accented e should be 2 bytes, so each entry is 6 bytes instead of 5
         values = [
           "('é')",
-          "('é')" ]
+          "('é')"
+        ]
 
         base_sql_size_in_bytes = 15
         max_bytes = 26
 
-        values_size_in_bytes = values.sum {|value| value.bytesize }
         value_sets = parser.parse values, reserved_bytes: base_sql_size_in_bytes, max_bytes: max_bytes
 
         assert_equal 2, value_sets.size, 'Two value sets were expected!'
       end
     end
   end
-
 end
-

data/test/value_sets_records_parser_test.rb

@@ -4,27 +4,27 @@ require 'activerecord-import/value_sets_parser'
 
 describe "ActiveRecord::Import::ValueSetsRecordsParser" do
   context "#parse - computing insert value sets" do
-    let(:parser){ ActiveRecord::Import::ValueSetsRecordsParser }
-    let(:base_sql){ "INSERT INTO atable (a,b,c)" }
-    let(:values){ [ "(1,2,3)", "(2,3,4)", "(3,4,5)" ] }
+    let(:parser) { ActiveRecord::Import::ValueSetsRecordsParser }
+    let(:base_sql) { "INSERT INTO atable (a,b,c)" }
+    let(:values) { ["(1,2,3)", "(2,3,4)", "(3,4,5)"] }
 
     context "when the max number of records is 1" do
       it "should return 3 value sets when given 3 values sets" do
-        value_sets = parser.parse values, :max_records => 1
+        value_sets = parser.parse values, max_records: 1
         assert_equal 3, value_sets.size
       end
     end
 
     context "when the max number of records is 2" do
       it "should return 2 value sets when given 3 values sets" do
-        value_sets = parser.parse values, :max_records => 2
+        value_sets = parser.parse values, max_records: 2
         assert_equal 2, value_sets.size
       end
     end
 
     context "when the max number of records is 3" do
       it "should return 1 value sets when given 3 values sets" do
-        value_sets = parser.parse values, :max_records => 3
+        value_sets = parser.parse values, max_records: 3
         assert_equal 1, value_sets.size
       end
     end