activerecord-import 0.23.0 → 1.4.0

data/lib/activerecord-import/synchronize.rb

@@ -39,8 +39,8 @@ module ActiveRecord # :nodoc:
 
         next unless matched_instance
 
-        instance.send :clear_aggregation_cache
-        instance.send :clear_association_cache
+        instance.instance_variable_set :@association_cache, {}
+        instance.send :clear_aggregation_cache if instance.respond_to?(:clear_aggregation_cache, true)
         instance.instance_variable_set :@attributes, matched_instance.instance_variable_get(:@attributes)
 
         if instance.respond_to?(:clear_changes_information)

data/lib/activerecord-import/value_sets_parser.rb

@@ -1,3 +1,5 @@
+require 'active_support/core_ext/array'
+
 module ActiveRecord::Import
   class ValueSetTooLargeError < StandardError
     attr_reader :size

data/lib/activerecord-import/version.rb

@@ -1,5 +1,5 @@
 module ActiveRecord
   module Import
-    VERSION = "0.23.0".freeze
+    VERSION = "1.4.0".freeze
   end
 end

data/lib/activerecord-import.rb

@@ -1,4 +1,5 @@
 # rubocop:disable Style/FileName
+require "active_support/lazy_load_hooks"
 
 ActiveSupport.on_load(:active_record) do
   require "activerecord-import/base"

data/test/adapters/makara_postgis.rb

@@ -0,0 +1 @@
+ENV["ARE_DB"] = "postgis"

data/test/{travis → github}/database.yml

@@ -1,7 +1,8 @@
 common: &common
   username: root
-  password:
+  password: root
   encoding: utf8
+  collation: utf8_general_ci
   host:     localhost
   database: activerecord_import_test
 
@@ -37,6 +38,7 @@ oracle:
 postgresql: &postgresql
   <<: *common
   username: postgres
+  password: postgres
   adapter: postgresql
   min_messages: warning
 

data/test/import_test.rb

@@ -17,6 +17,11 @@ describe "#import" do
     assert_equal error.message, "Last argument should be a two dimensional array '[[]]'. First element in array was a String"
   end
 
+  it "warns you that you're passing more data than you ought to" do
+    error = assert_raise(ArgumentError) { Topic.import %w(title author_name), [['Author #1', 'Book #1', 0]] }
+    assert_equal error.message, "Number of values (8) exceeds number of columns (7)"
+  end
+
   it "should not produce an error when importing empty arrays" do
     assert_nothing_raised do
       Topic.import []
@@ -164,7 +169,17 @@ describe "#import" do
       assert_difference "Dictionary.count", +1 do
         Dictionary.import dictionaries
       end
-      assert_equal "Dictionary", Dictionary.first.type
+      assert_equal "Dictionary", Dictionary.last.type
+    end
+
+    it "should import arrays successfully" do
+      columns = [:author_name, :title]
+      values =  [["Noah Webster", "Webster's Dictionary"]]
+
+      assert_difference "Dictionary.count", +1 do
+        Dictionary.import columns, values
+      end
+      assert_equal "Dictionary", Dictionary.last.type
     end
   end
 
@@ -210,9 +225,9 @@ describe "#import" do
       end
 
       it "should ignore uniqueness validators" do
-        Topic.import columns, valid_values, validate: true
+        Topic.import columns, valid_values
         assert_difference "Topic.count", +2 do
-          Topic.import columns, valid_values, validate: true
+          Topic.import columns, valid_values
         end
       end
 
@@ -247,6 +262,16 @@ describe "#import" do
         end
       end
 
+      it "should index the failed instances by their poistion in the set if `track_failures` is true" do
+        index_offset = valid_values.length
+        results = Topic.import columns, valid_values + invalid_values, validate: true, track_validation_failures: true
+        assert_equal invalid_values.size, results.failed_instances.size
+        invalid_values.each_with_index do |value_set, index|
+          assert_equal index + index_offset, results.failed_instances[index].first
+          assert_equal value_set.first, results.failed_instances[index].last.title
+        end
+      end
+
       it "should set ids in valid models if adapter supports setting primary key of imported objects" do
         if ActiveRecord::Base.supports_setting_primary_key_of_imported_objects?
           Topic.import (invalid_models + valid_models), validate: true
@@ -294,6 +319,36 @@ describe "#import" do
         end
       end
     end
+
+    context "with uniqueness validators included" do
+      it "should not import duplicate records" do
+        Topic.import columns, valid_values
+        assert_no_difference "Topic.count" do
+          Topic.import columns, valid_values, validate_uniqueness: true
+        end
+      end
+    end
+
+    context "when validatoring presence of belongs_to association" do
+      it "should not import records without foreign key" do
+        assert_no_difference "UserToken.count" do
+          UserToken.import [:token], [['12345abcdef67890']]
+        end
+      end
+
+      it "should import records with foreign key" do
+        assert_difference "UserToken.count", +1 do
+          UserToken.import [:user_name, :token], [%w("Bob", "12345abcdef67890")]
+        end
+      end
+
+      it "should not mutate the defined validations" do
+        UserToken.import [:user_name, :token], [%w("Bob", "12345abcdef67890")]
+        ut = UserToken.new
+        ut.valid?
+        assert_includes ut.errors.messages, :user
+      end
+    end
   end
 
   context "without :validation option" do
@@ -360,6 +415,15 @@ describe "#import" do
         assert_equal 3, result.num_inserts if Topic.supports_import?
       end
     end
+
+    it "should accept and call an optional callable to run after each batch" do
+      lambda_called = 0
+
+      my_proc = ->(_row_count, _batches, _batch, _duration) { lambda_called += 1 }
+      Topic.import Build(10, :topics), batch_size: 4, batch_progress: my_proc
+
+      assert_equal 3, lambda_called
+    end
   end
 
   context "with :synchronize option" do
@@ -490,11 +554,15 @@ describe "#import" do
 
     context "when the timestamps columns are present" do
       setup do
-        @existing_book = Book.create(title: "Fell", author_name: "Curry", publisher: "Bayer", created_at: 2.years.ago.utc, created_on: 2.years.ago.utc)
-        ActiveRecord::Base.default_timezone = :utc
+        @existing_book = Book.create(title: "Fell", author_name: "Curry", publisher: "Bayer", created_at: 2.years.ago.utc, created_on: 2.years.ago.utc, updated_at: 2.years.ago.utc, updated_on: 2.years.ago.utc)
+        if ActiveRecord.respond_to?(:default_timezone)
+          ActiveRecord.default_timezone = :utc
+        else
+          ActiveRecord::Base.default_timezone = :utc
+        end
         Timecop.freeze(time) do
           assert_difference "Book.count", +2 do
-            Book.import %w(title author_name publisher created_at created_on), [["LDAP", "Big Bird", "Del Rey", nil, nil], [@existing_book.title, @existing_book.author_name, @existing_book.publisher, @existing_book.created_at, @existing_book.created_on]]
+            Book.import %w(title author_name publisher created_at created_on updated_at updated_on), [["LDAP", "Big Bird", "Del Rey", nil, nil, nil, nil], [@existing_book.title, @existing_book.author_name, @existing_book.publisher, @existing_book.created_at, @existing_book.created_on, @existing_book.updated_at, @existing_book.updated_on]]
           end
         end
         @new_book, @existing_book = Book.last 2
@@ -523,6 +591,23 @@ describe "#import" do
       it "should set the updated_on column for new records" do
         assert_in_delta time.to_i, @new_book.updated_on.to_i, 1.second
       end
+
+      it "should not set the updated_at column for existing records" do
+        assert_equal 2.years.ago.utc.strftime("%Y:%d"), @existing_book.updated_at.strftime("%Y:%d")
+      end
+
+      it "should not set the updated_on column for existing records" do
+        assert_equal 2.years.ago.utc.strftime("%Y:%d"), @existing_book.updated_on.strftime("%Y:%d")
+      end
+
+      it "should not set the updated_at column on models if changed" do
+        timestamp = Time.now.utc
+        books = [
+          Book.new(author_name: "Foo", title: "Baz", created_at: timestamp, updated_at: timestamp)
+        ]
+        Book.import books
+        assert_equal timestamp.strftime("%Y:%d"), Book.last.updated_at.strftime("%Y:%d")
+      end
     end
 
     context "when a custom time zone is set" do
@@ -567,7 +652,7 @@ describe "#import" do
 
   context "importing through an association scope" do
     { has_many: :chapters, polymorphic: :discounts }.each do |association_type, association|
-      book   = FactoryGirl.create :book
+      book   = FactoryBot.create :book
       scope  = book.public_send association
       klass  = { chapters: Chapter, discounts: Discount }[association]
       column = { chapters: :title,  discounts: :amount  }[association]
@@ -590,18 +675,34 @@ describe "#import" do
           assert_equal [val1, val2], scope.map(&column).sort
         end
 
+        context "for cards and decks" do
+          it "works when the polymorphic name is different than base class name" do
+            deck = Deck.create(id: 1, name: 'test')
+            deck.cards.import [:id, :deck_type], [[1, 'PlayingCard']]
+            assert_equal deck.cards.first.deck_type, "PlayingCard"
+          end
+        end
+
         it "works importing array of hashes" do
           scope.import [{ column => val1 }, { column => val2 }]
 
           assert_equal [val1, val2], scope.map(&column).sort
         end
       end
+
+      it "works with a non-standard association primary key" do
+        user = User.create(id: 1, name: 'Solomon')
+        user.user_tokens.import [:id, :token], [[5, '12345abcdef67890']]
+
+        token = UserToken.find(5)
+        assert_equal 'Solomon', token.user_name
+      end
     end
   end
 
   context "importing model with polymorphic belongs_to" do
     it "works without error" do
-      book     = FactoryGirl.create :book
+      book     = FactoryBot.create :book
       discount = Discount.new(discountable: book)
 
       Discount.import([discount])
@@ -840,4 +941,33 @@ describe "#import" do
       end
     end
   end
+  describe "importing model with after_initialize callback" do
+    let(:columns) { %w(name size) }
+    let(:valid_values) { [%w("Deer", "Small"), %w("Monkey", "Medium")] }
+    let(:invalid_values) do
+      [
+        { name: "giraffe", size: "Large" },
+        { size: "Medium" } # name is missing
+      ]
+    end
+    context "with validation checks turned off" do
+      it "should import valid data" do
+        Animal.import(columns, valid_values, validate: false)
+        assert_equal 2, Animal.count
+      end
+      it "should raise ArgumentError" do
+        assert_raise(ArgumentError) { Animal.import(invalid_values, validate: false) }
+      end
+    end
+
+    context "with validation checks turned on" do
+      it "should import valid data" do
+        Animal.import(columns, valid_values, validate: true)
+        assert_equal 2, Animal.count
+      end
+      it "should raise ArgumentError" do
+        assert_raise(ArgumentError) { Animal.import(invalid_values, validate: true) }
+      end
+    end
+  end
 end

data/test/makara_postgis/import_test.rb

@@ -0,0 +1,8 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples')
+
+should_support_postgresql_import_functionality
+
+if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
+  should_support_postgresql_upsert_functionality
+end

data/test/models/animal.rb

@@ -0,0 +1,6 @@
+class Animal < ActiveRecord::Base
+  after_initialize :validate_name_presence, if: :new_record?
+  def validate_name_presence
+    raise ArgumentError if name.nil?
+  end
+end

data/test/models/card.rb

@@ -0,0 +1,3 @@
+class Card < ActiveRecord::Base
+  belongs_to :deck, polymorphic: true
+end

data/test/models/customer.rb

@@ -0,0 +1,6 @@
+class Customer < ActiveRecord::Base
+  has_many :orders,
+    inverse_of: :customer,
+    primary_key: %i(account_id id),
+    foreign_key: %i(account_id customer_id)
+end

data/test/models/deck.rb

@@ -0,0 +1,6 @@
+class Deck < ActiveRecord::Base
+  has_many :cards
+  def self.polymorphic_name
+    "PlayingCard"
+  end
+end

data/test/models/order.rb

@@ -0,0 +1,6 @@
+class Order < ActiveRecord::Base
+  belongs_to :customer,
+    inverse_of: :orders,
+    primary_key: %i(account_id id),
+    foreign_key: %i(account_id customer_id)
+end

data/test/models/playing_card.rb

@@ -0,0 +1,2 @@
+class PlayingCard < ActiveRecord::Base
+end

data/test/models/user.rb

@@ -1 +1,3 @@
-class User < ActiveRecord::Base; end
+class User < ActiveRecord::Base
+  has_many :user_tokens, primary_key: :name, foreign_key: :user_name
+end

data/test/models/user_token.rb

@@ -0,0 +1,4 @@
+class UserToken < ActiveRecord::Base
+  belongs_to :user, primary_key: :name, foreign_key: :user_name
+  validates :user, presence: true
+end

data/test/schema/generic_schema.rb

@@ -52,6 +52,20 @@ ActiveRecord::Schema.define do
     t.string :name
   end
 
+  create_table :cards, force: :cascade do |t|
+    t.string :name
+    t.string :deck_type
+    t.integer :deck_id
+  end
+
+  create_table :decks, force: :cascade do |t|
+    t.string :name
+  end
+
+  create_table :playing_cards, force: :cascade do |t|
+    t.string :name
+  end
+
   create_table :books, force: :cascade do |t|
     t.string :title, null: false
     t.string :publisher, null: false, default: 'Default Publisher'
@@ -164,6 +178,11 @@ ActiveRecord::Schema.define do
     t.integer :lock_version, null: false, default: 0
   end
 
+  create_table :user_tokens, force: :cascade do |t|
+    t.string :user_name, null: false
+    t.string :token, null: false
+  end
+
   create_table :accounts, force: :cascade do |t|
     t.string :name, null: false
     t.integer :lock, null: false, default: 0
@@ -186,4 +205,15 @@ ActiveRecord::Schema.define do
       );
     ).split.join(' ').strip
   end
+
+  create_table :customers, force: :cascade do |t|
+    t.integer :account_id
+    t.string :name
+  end
+
+  create_table :orders, force: :cascade do |t|
+    t.integer :account_id
+    t.integer :customer_id
+    t.integer :amount
+  end
 end

data/test/schema/mysql2_schema.rb

@@ -0,0 +1,19 @@
+ActiveRecord::Schema.define do
+  create_table :books, force: :cascade do |t|
+    t.string :title, null: false
+    t.virtual :upper_title, type: :string, as: "upper(`title`)" if t.respond_to?(:virtual)
+    t.string :publisher, null: false, default: 'Default Publisher'
+    t.string :author_name, null: false
+    t.datetime :created_at
+    t.datetime :created_on
+    t.datetime :updated_at
+    t.datetime :updated_on
+    t.date :publish_date
+    t.integer :topic_id
+    t.integer :tag_id
+    t.integer :publisher_id
+    t.boolean :for_sale, default: true
+    t.integer :status, default: 0
+    t.string :type
+  end
+end

data/test/schema/postgresql_schema.rb

@@ -3,11 +3,24 @@ ActiveRecord::Schema.define do
   execute('CREATE extension IF NOT EXISTS "pgcrypto";')
   execute('CREATE extension IF NOT EXISTS "uuid-ossp";')
 
+  # create ENUM if it does not exist yet
+  begin
+    execute('CREATE TYPE vendor_type AS ENUM (\'wholesaler\', \'retailer\');')
+  rescue ActiveRecord::StatementInvalid => e
+    # since PostgreSQL does not support IF NOT EXISTS when creating a TYPE,
+    # rescue the error and check the error class
+    raise unless e.cause.is_a? PG::DuplicateObject
+    execute('ALTER TYPE vendor_type ADD VALUE IF NOT EXISTS \'wholesaler\';')
+    execute('ALTER TYPE vendor_type ADD VALUE IF NOT EXISTS \'retailer\';')
+  end
+
   create_table :vendors, id: :uuid, force: :cascade do |t|
     t.string :name, null: true
+    t.text :hours
     t.text :preferences
 
     if t.respond_to?(:json)
+      t.json :pure_json_data
       t.json :data
     else
       t.text :data
@@ -20,6 +33,7 @@ ActiveRecord::Schema.define do
     end
 
     if t.respond_to?(:jsonb)
+      t.jsonb :pure_jsonb_data
       t.jsonb :settings
       t.jsonb :json_data, null: false, default: {}
     else
@@ -27,6 +41,8 @@ ActiveRecord::Schema.define do
       t.text :json_data
     end
 
+    t.column :vendor_type, :vendor_type
+
     t.datetime :created_at
     t.datetime :updated_at
   end

data/test/schema/sqlite3_schema.rb

@@ -0,0 +1,13 @@
+ActiveRecord::Schema.define do
+  create_table :alarms, force: true do |t|
+    t.column :device_id, :integer, null: false
+    t.column :alarm_type, :integer, null: false
+    t.column :status, :integer, null: false
+    t.column :metadata, :text
+    t.column :secret_key, :binary
+    t.datetime :created_at
+    t.datetime :updated_at
+  end
+
+  add_index :alarms, [:device_id, :alarm_type], unique: true, where: 'status <> 0'
+end

data/test/support/factories.rb

@@ -1,4 +1,4 @@
-FactoryGirl.define do
+FactoryBot.define do
   sequence(:book_title) { |n| "Book #{n}" }
   sequence(:chapter_title) { |n| "Chapter #{n}" }
   sequence(:end_note) { |n| "Endnote #{n}" }
@@ -9,7 +9,7 @@ FactoryGirl.define do
 
   factory :invalid_topic, class: "Topic" do
     sequence(:title) { |n| "Title #{n}" }
-    author_name nil
+    author_name { nil }
   end
 
   factory :topic do
@@ -27,7 +27,7 @@ FactoryGirl.define do
 
     trait :with_rule do
       after(:build) do |question|
-        question.build_rule(FactoryGirl.attributes_for(:rule))
+        question.build_rule(FactoryBot.attributes_for(:rule))
       end
     end
   end
@@ -40,21 +40,21 @@ FactoryGirl.define do
   factory :topic_with_book, parent: :topic do
     after(:build) do |topic|
       2.times do
-        book = topic.books.build(title: FactoryGirl.generate(:book_title), author_name: 'Stephen King')
+        book = topic.books.build(title: FactoryBot.generate(:book_title), author_name: 'Stephen King')
         3.times do
-          book.chapters.build(title: FactoryGirl.generate(:chapter_title))
+          book.chapters.build(title: FactoryBot.generate(:chapter_title))
         end
 
         4.times do
-          book.end_notes.build(note: FactoryGirl.generate(:end_note))
+          book.end_notes.build(note: FactoryBot.generate(:end_note))
         end
       end
     end
   end
 
   factory :book do
-    title 'Tortilla Flat'
-    author_name 'John Steinbeck'
+    title { 'Tortilla Flat' }
+    author_name { 'John Steinbeck' }
   end
 
   factory :car do

data/test/support/generate.rb

@@ -2,28 +2,28 @@ class ActiveSupport::TestCase
   def Build(*args) # rubocop:disable Style/MethodName
     n = args.shift if args.first.is_a?(Numeric)
     factory = args.shift
-    factory_girl_args = args.shift || {}
+    factory_bot_args = args.shift || {}
 
     if n
       [].tap do |collection|
-        n.times.each { collection << FactoryGirl.build(factory.to_s.singularize.to_sym, factory_girl_args) }
+        n.times.each { collection << FactoryBot.build(factory.to_s.singularize.to_sym, factory_bot_args) }
       end
     else
-      FactoryGirl.build(factory.to_s.singularize.to_sym, factory_girl_args)
+      FactoryBot.build(factory.to_s.singularize.to_sym, factory_bot_args)
     end
   end
 
   def Generate(*args) # rubocop:disable Style/MethodName
     n = args.shift if args.first.is_a?(Numeric)
     factory = args.shift
-    factory_girl_args = args.shift || {}
+    factory_bot_args = args.shift || {}
 
     if n
       [].tap do |collection|
-        n.times.each { collection << FactoryGirl.create(factory.to_s.singularize.to_sym, factory_girl_args) }
+        n.times.each { collection << FactoryBot.create(factory.to_s.singularize.to_sym, factory_bot_args) }
       end
     else
-      FactoryGirl.create(factory.to_s.singularize.to_sym, factory_girl_args)
+      FactoryBot.create(factory.to_s.singularize.to_sym, factory_bot_args)
     end
   end
 end

data/test/support/mysql/import_examples.rb

@@ -82,5 +82,17 @@ def should_support_mysql_import_functionality
         assert_equal "Chad Fowler", topics.last.author_name, "wrong author!"
       end
     end
+
+    if ENV['AR_VERSION'].to_f >= 5.1
+      context "with virtual columns" do
+        let(:books) { [Book.new(author_name: "foo", title: "bar")] }
+
+        it "ignores virtual columns and creates record" do
+          assert_difference "Book.count", +1 do
+            Book.import books
+          end
+        end
+      end
+    end
   end
 end