activerecord-import 1.0.3

data/test/support/sqlite3/import_examples.rb

@@ -0,0 +1,231 @@
+# encoding: UTF-8
+def should_support_sqlite3_import_functionality
+  if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
+    should_support_sqlite_upsert_functionality
+  end
+
+  describe "#supports_imports?" do
+    it "should support import" do
+      assert ActiveRecord::Base.supports_import?
+    end
+  end
+
+  describe "#import" do
+    it "imports with a single insert on SQLite 3.7.11 or higher" do
+      assert_difference "Topic.count", +507 do
+        result = Topic.import Build(7, :topics)
+        assert_equal 1, result.num_inserts, "Failed to issue a single INSERT statement. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+        assert_equal 7, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+
+        result = Topic.import Build(500, :topics)
+        assert_equal 1, result.num_inserts, "Failed to issue a single INSERT statement. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+        assert_equal 507, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+      end
+    end
+
+    it "imports with a two inserts on SQLite 3.7.11 or higher" do
+      assert_difference "Topic.count", +501 do
+        result = Topic.import Build(501, :topics)
+        assert_equal 2, result.num_inserts, "Failed to issue a two INSERT statements. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+        assert_equal 501, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+      end
+    end
+
+    it "imports with a five inserts on SQLite 3.7.11 or higher" do
+      assert_difference "Topic.count", +2500 do
+        result = Topic.import Build(2500, :topics)
+        assert_equal 5, result.num_inserts, "Failed to issue a two INSERT statements. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+        assert_equal 2500, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
+      end
+    end
+  end
+end
+
+def should_support_sqlite_upsert_functionality
+  should_support_basic_on_duplicate_key_update
+  should_support_on_duplicate_key_ignore
+
+  describe "#import" do
+    extend ActiveSupport::TestCase::ImportAssertions
+
+    macro(:perform_import) { raise "supply your own #perform_import in a context below" }
+    macro(:updated_topic) { Topic.find(@topic.id) }
+
+    context "with :on_duplicate_key_ignore and validation checks turned off" do
+      let(:columns) { %w( id title author_name author_email_address parent_id ) }
+      let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] }
+      let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] }
+
+      setup do
+        Topic.import columns, values, validate: false
+      end
+
+      it "should not update any records" do
+        result = Topic.import columns, updated_values, on_duplicate_key_ignore: true, validate: false
+        assert_equal [], result.ids
+      end
+    end
+
+    context "with :on_duplicate_key_update and validation checks turned off" do
+      asssertion_group(:should_support_on_duplicate_key_update) do
+        should_not_update_fields_not_mentioned
+        should_update_foreign_keys
+        should_not_update_created_at_on_timestamp_columns
+        should_update_updated_at_on_timestamp_columns
+      end
+
+      context "using a hash" do
+        context "with :columns a hash" do
+          let(:columns) { %w( id title author_name author_email_address parent_id ) }
+          let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] }
+          let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] }
+
+          macro(:perform_import) do |*opts|
+            Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id, columns: update_columns }, validate: false)
+          end
+
+          setup do
+            Topic.import columns, values, validate: false
+            @topic = Topic.find 99
+          end
+
+          it "should not modify the passed in :on_duplicate_key_update columns array" do
+            assert_nothing_raised do
+              columns = %w(title author_name).freeze
+              Topic.import columns, [%w(foo, bar)], on_duplicate_key_update: { columns: columns }
+            end
+          end
+
+          context "using string hash map" do
+            let(:update_columns) { { "title" => "title", "author_email_address" => "author_email_address", "parent_id" => "parent_id" } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned
+          end
+
+          context "using string hash map, but specifying column mismatches" do
+            let(:update_columns) { { "title" => "author_email_address", "author_email_address" => "title", "parent_id" => "parent_id" } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned_with_hash_mappings
+          end
+
+          context "using symbol hash map" do
+            let(:update_columns) { { title: :title, author_email_address: :author_email_address, parent_id: :parent_id } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned
+          end
+
+          context "using symbol hash map, but specifying column mismatches" do
+            let(:update_columns) { { title: :author_email_address, author_email_address: :title, parent_id: :parent_id } }
+            should_support_on_duplicate_key_update
+            should_update_fields_mentioned_with_hash_mappings
+          end
+        end
+
+        context 'with :index_predicate' do
+          let(:columns) { %w( id device_id alarm_type status metadata ) }
+          let(:values) { [[99, 17, 1, 1, 'foo']] }
+          let(:updated_values) { [[99, 17, 1, 2, 'bar']] }
+
+          macro(:perform_import) do |*opts|
+            Alarm.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: [:device_id, :alarm_type], index_predicate: 'status <> 0', columns: [:status] }, validate: false)
+          end
+
+          macro(:updated_alarm) { Alarm.find(@alarm.id) }
+
+          setup do
+            Alarm.import columns, values, validate: false
+            @alarm = Alarm.find 99
+          end
+
+          context 'supports on duplicate key update for partial indexes' do
+            it 'should not update created_at timestamp columns' do
+              Timecop.freeze Chronic.parse("5 minutes from now") do
+                perform_import
+                assert_in_delta @alarm.created_at.to_i, updated_alarm.created_at.to_i, 1
+              end
+            end
+
+            it 'should update updated_at timestamp columns' do
+              time = Chronic.parse("5 minutes from now")
+              Timecop.freeze time do
+                perform_import
+                assert_in_delta time.to_i, updated_alarm.updated_at.to_i, 1
+              end
+            end
+
+            it 'should not update fields not mentioned' do
+              perform_import
+              assert_equal 'foo', updated_alarm.metadata
+            end
+
+            it 'should update fields mentioned with hash mappings' do
+              perform_import
+              assert_equal 2, updated_alarm.status
+            end
+          end
+        end
+
+        context 'with :condition' do
+          let(:columns) { %w( id device_id alarm_type status metadata) }
+          let(:values) { [[99, 17, 1, 1, 'foo']] }
+          let(:updated_values) { [[99, 17, 1, 1, 'bar']] }
+
+          macro(:perform_import) do |*opts|
+            Alarm.import(
+              columns,
+              updated_values,
+              opts.extract_options!.merge(
+                on_duplicate_key_update: {
+                  conflict_target: [:id],
+                  condition: "alarms.metadata NOT LIKE '%foo%'",
+                  columns: [:metadata]
+                },
+                validate: false
+              )
+            )
+          end
+
+          macro(:updated_alarm) { Alarm.find(@alarm.id) }
+
+          setup do
+            Alarm.import columns, values, validate: false
+            @alarm = Alarm.find 99
+          end
+
+          it 'should not update fields not matched' do
+            perform_import
+            assert_equal 'foo', updated_alarm.metadata
+          end
+        end
+
+        context "with no :conflict_target" do
+          context "with no primary key" do
+            it "raises ArgumentError" do
+              error = assert_raises ArgumentError do
+                Rule.import Build(3, :rules), on_duplicate_key_update: [:condition_text], validate: false
+              end
+              assert_match(/Expected :conflict_target to be specified/, error.message)
+            end
+          end
+        end
+
+        context "with no :columns" do
+          let(:columns) { %w( id title author_name author_email_address ) }
+          let(:values) { [[100, "Book", "John Doe", "john@doe.com"]] }
+          let(:updated_values) { [[100, "Title Should Not Change", "Author Should Not Change", "john@nogo.com"]] }
+
+          macro(:perform_import) do |*opts|
+            Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id }, validate: false)
+          end
+
+          setup do
+            Topic.import columns, values, validate: false
+            @topic = Topic.find 100
+          end
+
+          should_update_updated_at_on_timestamp_columns
+        end
+      end
+    end
+  end
+end

data/test/synchronize_test.rb

@@ -0,0 +1,41 @@
+require File.expand_path('../test_helper', __FILE__)
+
+describe ".synchronize" do
+  let(:topics) { Generate(3, :topics) }
+  let(:titles) { %w(one two three) }
+
+  setup do
+    # update records outside of ActiveRecord knowing about it
+    Topic.connection.execute( "UPDATE #{Topic.table_name} SET title='#{titles[0]}_haha' WHERE id=#{topics[0].id}", "Updating record 1 without ActiveRecord" )
+    Topic.connection.execute( "UPDATE #{Topic.table_name} SET title='#{titles[1]}_haha' WHERE id=#{topics[1].id}", "Updating record 2 without ActiveRecord" )
+    Topic.connection.execute( "UPDATE #{Topic.table_name} SET title='#{titles[2]}_haha' WHERE id=#{topics[2].id}", "Updating record 3 without ActiveRecord" )
+  end
+
+  it "reloads data for the specified records" do
+    Topic.synchronize topics
+
+    actual_titles = topics.map(&:title)
+    assert_equal "#{titles[0]}_haha", actual_titles[0], "the first record was not correctly updated"
+    assert_equal "#{titles[1]}_haha", actual_titles[1], "the second record was not correctly updated"
+    assert_equal "#{titles[2]}_haha", actual_titles[2], "the third record was not correctly updated"
+  end
+
+  it "the synchronized records aren't dirty" do
+    # Update the in memory records so they're dirty
+    topics.each { |topic| topic.title = 'dirty title' }
+
+    Topic.synchronize topics
+
+    assert_equal false, topics[0].changed?, "the first record was dirty"
+    assert_equal false, topics[1].changed?, "the second record was dirty"
+    assert_equal false, topics[2].changed?, "the third record was dirty"
+  end
+
+  it "ignores default scope" do
+    # update records outside of ActiveRecord knowing about it
+    Topic.connection.execute( "UPDATE #{Topic.table_name} SET approved='0' WHERE id=#{topics[0].id}", "Updating record 1 without ActiveRecord" )
+
+    Topic.synchronize topics
+    assert_equal false, topics[0].approved
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,75 @@
+require 'pathname'
+test_dir = Pathname.new File.dirname(__FILE__)
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require "fileutils"
+
+ENV["RAILS_ENV"] = "test"
+
+require "bundler"
+Bundler.setup
+
+require 'pry' unless RbConfig::CONFIG["RUBY_INSTALL_NAME"] =~ /jruby/
+
+require "active_record"
+require "active_record/fixtures"
+require "active_support/test_case"
+
+if ActiveSupport::VERSION::STRING < "4.0"
+  require 'test/unit'
+  require 'mocha/test_unit'
+else
+  require 'active_support/testing/autorun'
+  require "mocha/mini_test"
+end
+
+require 'timecop'
+require 'chronic'
+
+begin
+  require 'composite_primary_keys'
+rescue LoadError
+  ENV["SKIP_COMPOSITE_PK"] = "true"
+end
+
+# Support MySQL 5.7
+if ActiveSupport::VERSION::STRING < "4.1"
+  require "active_record/connection_adapters/mysql2_adapter"
+  class ActiveRecord::ConnectionAdapters::Mysql2Adapter
+    NATIVE_DATABASE_TYPES[:primary_key] = "int(11) auto_increment PRIMARY KEY"
+  end
+end
+
+require "ruby-debug" if RUBY_VERSION.to_f < 1.9
+
+adapter = ENV["ARE_DB"] || "sqlite3"
+
+FileUtils.mkdir_p 'log'
+ActiveRecord::Base.logger = Logger.new("log/test.log")
+ActiveRecord::Base.logger.level = Logger::DEBUG
+ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
+ActiveRecord::Base.default_timezone = :utc
+
+require "activerecord-import"
+ActiveRecord::Base.establish_connection :test
+
+ActiveSupport::Notifications.subscribe(/active_record.sql/) do |_, _, _, _, hsh|
+  ActiveRecord::Base.logger.info hsh[:sql]
+end
+
+require "factory_bot"
+Dir[File.dirname(__FILE__) + "/support/**/*.rb"].each { |file| require file }
+
+# Load base/generic schema
+require test_dir.join("schema/version")
+require test_dir.join("schema/generic_schema")
+adapter_schema = test_dir.join("schema/#{adapter}_schema.rb")
+require adapter_schema if File.exist?(adapter_schema)
+
+Dir[File.dirname(__FILE__) + "/models/*.rb"].each { |file| require file }
+
+# Prevent this deprecation warning from breaking the tests.
+Rake::FileList.send(:remove_method, :import)
+
+ActiveSupport::TestCase.test_order = :random if ENV['AR_VERSION'].to_f >= 4.2

data/test/travis/database.yml

@@ -0,0 +1,66 @@
+common: &common
+  username: root
+  password:
+  encoding: utf8
+  host:     localhost
+  database: activerecord_import_test
+
+jdbcpostgresql: &postgresql
+  <<: *common
+  username: postgres
+  adapter: jdbcpostgresql
+  min_messages: warning
+
+jdbcmysql: &mysql2
+  <<: *common
+  adapter: jdbcmysql
+
+jdbcsqlite3: &sqlite3
+  <<: *common
+  adapter: jdbcsqlite3
+
+mysql2: &mysql2
+  <<: *common
+  adapter: mysql2
+
+mysql2spatial:
+  <<: *mysql2
+
+mysql2_makara:
+  <<: *mysql2
+
+oracle:
+  <<: *common
+  adapter: oracle
+  min_messages: debug
+
+postgresql: &postgresql
+  <<: *common
+  username: postgres
+  adapter: postgresql
+  min_messages: warning
+
+postresql_makara:
+  <<: *postgresql
+
+postgis:
+  <<: *postgresql
+
+seamless_database_pool:
+  <<: *common
+  adapter: seamless_database_pool
+  pool_adapter: mysql2
+  prepared_statements: false
+  master:
+    host: localhost
+
+sqlite:
+  adapter: sqlite
+  dbfile: test.db
+
+sqlite3: &sqlite3
+  adapter: sqlite3
+  database: ":memory:"
+
+spatialite:
+  <<: *sqlite3

data/test/value_sets_bytes_parser_test.rb

@@ -0,0 +1,104 @@
+require File.expand_path(File.dirname(__FILE__) + '/test_helper')
+
+require 'activerecord-import/value_sets_parser'
+
+describe ActiveRecord::Import::ValueSetsBytesParser do
+  context "#parse - computing insert value sets" do
+    let(:parser) { ActiveRecord::Import::ValueSetsBytesParser }
+    let(:base_sql) { "INSERT INTO atable (a,b,c)" }
+    let(:values) { ["(1,2,3)", "(2,3,4)", "(3,4,5)"] }
+
+    context "when the max allowed bytes is 30 and the base SQL is 26 bytes" do
+      it "should raise ActiveRecord::Import::ValueSetTooLargeError" do
+        error = assert_raises ActiveRecord::Import::ValueSetTooLargeError do
+          parser.parse values, reserved_bytes: base_sql.size, max_bytes: 30
+        end
+        assert_match(/33 bytes exceeds the max allowed for an insert \[30\]/, error.message)
+      end
+    end
+
+    context "when the max allowed bytes is 33 and the base SQL is 26 bytes" do
+      it "should return 3 value sets when given 3 value sets of 7 bytes a piece" do
+        value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 33
+        assert_equal 3, value_sets.size
+      end
+    end
+
+    context "when the max allowed bytes is 40 and the base SQL is 26 bytes" do
+      it "should return 3 value sets when given 3 value sets of 7 bytes a piece" do
+        value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 40
+        assert_equal 3, value_sets.size
+      end
+    end
+
+    context "when the max allowed bytes is 41 and the base SQL is 26 bytes" do
+      it "should return 2 value sets when given 2 value sets of 7 bytes a piece" do
+        value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 41
+        assert_equal 2, value_sets.size
+      end
+    end
+
+    context "when the max allowed bytes is 48 and the base SQL is 26 bytes" do
+      it "should return 2 value sets when given 2 value sets of 7 bytes a piece" do
+        value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 48
+        assert_equal 2, value_sets.size
+      end
+    end
+
+    context "when the max allowed bytes is 49 and the base SQL is 26 bytes" do
+      it "should return 1 value sets when given 1 value sets of 7 bytes a piece" do
+        value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 49
+        assert_equal 1, value_sets.size
+      end
+    end
+
+    context "when the max allowed bytes is 999999 and the base SQL is 26 bytes" do
+      it "should return 1 value sets when given 1 value sets of 7 bytes a piece" do
+        value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 999_999
+        assert_equal 1, value_sets.size
+      end
+    end
+
+    it "should properly build insert value set based on max packet allowed" do
+      values = [
+        "('1','2','3')",
+        "('4','5','6')",
+        "('7','8','9')"
+      ]
+
+      base_sql_size_in_bytes = 15
+      max_bytes = 30
+
+      value_sets = parser.parse values, reserved_bytes: base_sql_size_in_bytes, max_bytes: max_bytes
+      assert_equal 3, value_sets.size, 'Three value sets were expected!'
+
+      # Each element in the value_sets array must be an array
+      value_sets.each_with_index do |e, i|
+        assert_kind_of Array, e, "Element #{i} was expected to be an Array!"
+      end
+
+      # Each element in the values array should have a 1:1 correlation to the elements
+      # in the returned value_sets arrays
+      assert_equal values[0], value_sets[0].first
+      assert_equal values[1], value_sets[1].first
+      assert_equal values[2], value_sets[2].first
+    end
+
+    context "data contains multi-byte chars" do
+      it "should properly build insert value set based on max packet allowed" do
+        # each accented e should be 2 bytes, so each entry is 6 bytes instead of 5
+        values = [
+          "('é')",
+          "('é')"
+        ]
+
+        base_sql_size_in_bytes = 15
+        max_bytes = 26
+
+        value_sets = parser.parse values, reserved_bytes: base_sql_size_in_bytes, max_bytes: max_bytes
+
+        assert_equal 2, value_sets.size, 'Two value sets were expected!'
+      end
+    end
+  end
+end