activerecord-import 0.19.0 → 1.0.0

data/lib/activerecord-import/synchronize.rb

@@ -31,7 +31,7 @@ module ActiveRecord # :nodoc:
 
       klass = instances.first.class
 
-      fresh_instances = klass.where(conditions).order(order)
+      fresh_instances = klass.unscoped.where(conditions).order(order)
       instances.each do |instance|
         matched_instance = fresh_instances.detect do |fresh_instance|
           keys.all? { |key| fresh_instance.send(key) == instance.send(key) }

data/lib/activerecord-import/value_sets_parser.rb

@@ -1,4 +1,12 @@
 module ActiveRecord::Import
+  class ValueSetTooLargeError < StandardError
+    attr_reader :size
+    def initialize(msg = "Value set exceeds max size", size = 0)
+      @size = size
+      super(msg)
+    end
+  end
+
   class ValueSetsBytesParser
     attr_reader :reserved_bytes, :max_bytes, :values
 
@@ -18,6 +26,12 @@ module ActiveRecord::Import
       current_size = 0
       values.each_with_index do |val, i|
         comma_bytes = arr.size
+        insert_size = reserved_bytes + val.bytesize
+
+        if insert_size > max_bytes
+          raise ValueSetTooLargeError.new("#{insert_size} bytes exceeds the max allowed for an insert [#{@max_bytes}]", insert_size)
+        end
+
         bytes_thus_far = reserved_bytes + current_size + val.bytesize + comma_bytes
         if bytes_thus_far <= max_bytes
           current_size += val.bytesize

data/lib/activerecord-import/version.rb

@@ -1,5 +1,5 @@
 module ActiveRecord
   module Import
-    VERSION = "0.19.0".freeze
+    VERSION = "1.0.0".freeze
   end
 end

data/lib/activerecord-import.rb

@@ -1,19 +1,6 @@
 # rubocop:disable Style/FileName
+require "active_support/lazy_load_hooks"
 
 ActiveSupport.on_load(:active_record) do
-  class ActiveRecord::Base
-    class << self
-      def establish_connection_with_activerecord_import(*args)
-        conn = establish_connection_without_activerecord_import(*args)
-        if !ActiveRecord.const_defined?(:Import) || !ActiveRecord::Import.respond_to?(:load_from_connection_pool)
-          require "activerecord-import/base"
-        end
-
-        ActiveRecord::Import.load_from_connection_pool connection_pool
-        conn
-      end
-      alias establish_connection_without_activerecord_import establish_connection
-      alias establish_connection establish_connection_with_activerecord_import
-    end
-  end
+  require "activerecord-import/base"
 end

data/test/adapters/makara_postgis.rb

@@ -0,0 +1 @@
+ENV["ARE_DB"] = "postgis"

data/test/import_test.rb

@@ -86,6 +86,54 @@ describe "#import" do
         assert_nil t.author_email_address
       end
     end
+
+    context "with extra keys" do
+      let(:values) do
+        [
+          { title: "LDAP", author_name: "Jerry Carter" },
+          { title: "Rails Recipes", author_name: "Chad Fowler", author_email_address: "cfowler@test.com" } # author_email_address is unknown
+        ]
+      end
+
+      it "should fail when column names are not specified" do
+        err = assert_raises ArgumentError do
+          Topic.import values, validate: false
+        end
+
+        assert err.message.include? 'Extra keys: [:author_email_address]'
+      end
+
+      it "should succeed when column names are specified" do
+        assert_difference "Topic.count", +2 do
+          Topic.import columns, values, validate: false
+        end
+      end
+    end
+
+    context "with missing keys" do
+      let(:values) do
+        [
+          { title: "LDAP", author_name: "Jerry Carter" },
+          { title: "Rails Recipes" } # author_name is missing
+        ]
+      end
+
+      it "should fail when column names are not specified" do
+        err = assert_raises ArgumentError do
+          Topic.import values, validate: false
+        end
+
+        assert err.message.include? 'Missing keys: [:author_name]'
+      end
+
+      it "should fail on missing hash key from specified column names" do
+        err = assert_raises ArgumentError do
+          Topic.import %i(author_name), values, validate: false
+        end
+
+        assert err.message.include? 'Missing keys: [:author_name]'
+      end
+    end
   end
 
   unless ENV["SKIP_COMPOSITE_PK"]
@@ -121,12 +169,12 @@ describe "#import" do
   end
 
   context "with :validation option" do
-    let(:columns) { %w(title author_name) }
-    let(:valid_values) { [["LDAP", "Jerry Carter"], ["Rails Recipes", "Chad Fowler"]] }
-    let(:valid_values_with_context) { [[1111, "Jerry Carter"], [2222, "Chad Fowler"]] }
-    let(:invalid_values) { [["The RSpec Book", ""], ["Agile+UX", ""]] }
-    let(:valid_models) { valid_values.map { |title, author_name| Topic.new(title: title, author_name: author_name) } }
-    let(:invalid_models) { invalid_values.map { |title, author_name| Topic.new(title: title, author_name: author_name) } }
+    let(:columns) { %w(title author_name content) }
+    let(:valid_values) { [["LDAP", "Jerry Carter", "Putting Directories to Work."], ["Rails Recipes", "Chad Fowler", "A trusted collection of solutions."]] }
+    let(:valid_values_with_context) { [[1111, "Jerry Carter", "1111"], [2222, "Chad Fowler", "2222"]] }
+    let(:invalid_values) { [["The RSpec Book", "David Chelimsky", "..."], ["Agile+UX", "", "All about Agile in UX."]] }
+    let(:valid_models) { valid_values.map { |title, author_name, content| Topic.new(title: title, author_name: author_name, content: content) } }
+    let(:invalid_models) { invalid_values.map { |title, author_name, content| Topic.new(title: title, author_name: author_name, content: content) } }
 
     context "with validation checks turned off" do
       it "should import valid data" do
@@ -162,9 +210,9 @@ describe "#import" do
       end
 
       it "should ignore uniqueness validators" do
-        Topic.import columns, valid_values, validate: true
+        Topic.import columns, valid_values
         assert_difference "Topic.count", +2 do
-          Topic.import columns, valid_values, validate: true
+          Topic.import columns, valid_values
         end
       end
 
@@ -200,7 +248,7 @@ describe "#import" do
       end
 
       it "should set ids in valid models if adapter supports setting primary key of imported objects" do
-        if ActiveRecord::Base.support_setting_primary_key_of_imported_objects?
+        if ActiveRecord::Base.supports_setting_primary_key_of_imported_objects?
           Topic.import (invalid_models + valid_models), validate: true
           assert_nil invalid_models[0].id
           assert_nil invalid_models[1].id
@@ -210,7 +258,7 @@ describe "#import" do
       end
 
       it "should set ActiveRecord timestamps in valid models if adapter supports setting primary key of imported objects" do
-        if ActiveRecord::Base.support_setting_primary_key_of_imported_objects?
+        if ActiveRecord::Base.supports_setting_primary_key_of_imported_objects?
           Timecop.freeze(Time.at(0)) do
             Topic.import (invalid_models + valid_models), validate: true
           end
@@ -246,6 +294,36 @@ describe "#import" do
         end
       end
     end
+
+    context "with uniqueness validators included" do
+      it "should not import duplicate records" do
+        Topic.import columns, valid_values
+        assert_no_difference "Topic.count" do
+          Topic.import columns, valid_values, validate_uniqueness: true
+        end
+      end
+    end
+
+    context "when validatoring presence of belongs_to association" do
+      it "should not import records without foreign key" do
+        assert_no_difference "UserToken.count" do
+          UserToken.import [:token], [['12345abcdef67890']]
+        end
+      end
+
+      it "should import records with foreign key" do
+        assert_difference "UserToken.count", +1 do
+          UserToken.import [:user_name, :token], [%w("Bob", "12345abcdef67890")]
+        end
+      end
+
+      it "should not mutate the defined validations" do
+        UserToken.import [:user_name, :token], [%w("Bob", "12345abcdef67890")]
+        ut = UserToken.new
+        ut.valid?
+        assert_includes ut.errors.messages, :user
+      end
+    end
   end
 
   context "without :validation option" do
@@ -320,7 +398,7 @@ describe "#import" do
 
       it "doesn't reload any data (doesn't work)" do
         Topic.import new_topics, synchronize: new_topics
-        if Topic.support_setting_primary_key_of_imported_objects?
+        if Topic.supports_setting_primary_key_of_imported_objects?
           assert new_topics.all?(&:persisted?), "Records should have been reloaded"
         else
           assert new_topics.all?(&:new_record?), "No record should have been reloaded"
@@ -442,11 +520,11 @@ describe "#import" do
 
     context "when the timestamps columns are present" do
       setup do
-        @existing_book = Book.create(title: "Fell", author_name: "Curry", publisher: "Bayer", created_at: 2.years.ago.utc, created_on: 2.years.ago.utc)
+        @existing_book = Book.create(title: "Fell", author_name: "Curry", publisher: "Bayer", created_at: 2.years.ago.utc, created_on: 2.years.ago.utc, updated_at: 2.years.ago.utc, updated_on: 2.years.ago.utc)
         ActiveRecord::Base.default_timezone = :utc
         Timecop.freeze(time) do
           assert_difference "Book.count", +2 do
-            Book.import %w(title author_name publisher created_at created_on), [["LDAP", "Big Bird", "Del Rey", nil, nil], [@existing_book.title, @existing_book.author_name, @existing_book.publisher, @existing_book.created_at, @existing_book.created_on]]
+            Book.import %w(title author_name publisher created_at created_on updated_at updated_on), [["LDAP", "Big Bird", "Del Rey", nil, nil, nil, nil], [@existing_book.title, @existing_book.author_name, @existing_book.publisher, @existing_book.created_at, @existing_book.created_on, @existing_book.updated_at, @existing_book.updated_on]]
           end
         end
         @new_book, @existing_book = Book.last 2
@@ -475,6 +553,23 @@ describe "#import" do
       it "should set the updated_on column for new records" do
         assert_in_delta time.to_i, @new_book.updated_on.to_i, 1.second
       end
+
+      it "should not set the updated_at column for existing records" do
+        assert_equal 2.years.ago.utc.strftime("%Y:%d"), @existing_book.updated_at.strftime("%Y:%d")
+      end
+
+      it "should not set the updated_on column for existing records" do
+        assert_equal 2.years.ago.utc.strftime("%Y:%d"), @existing_book.updated_on.strftime("%Y:%d")
+      end
+
+      it "should not set the updated_at column on models if changed" do
+        timestamp = Time.now.utc
+        books = [
+          Book.new(author_name: "Foo", title: "Baz", created_at: timestamp, updated_at: timestamp)
+        ]
+        Book.import books
+        assert_equal timestamp.strftime("%Y:%d"), Book.last.updated_at.strftime("%Y:%d")
+      end
     end
 
     context "when a custom time zone is set" do
@@ -519,7 +614,7 @@ describe "#import" do
 
   context "importing through an association scope" do
     { has_many: :chapters, polymorphic: :discounts }.each do |association_type, association|
-      book   = FactoryGirl.create :book
+      book   = FactoryBot.create :book
       scope  = book.public_send association
       klass  = { chapters: Chapter, discounts: Discount }[association]
       column = { chapters: :title,  discounts: :amount  }[association]
@@ -541,10 +636,35 @@ describe "#import" do
 
           assert_equal [val1, val2], scope.map(&column).sort
         end
+
+        it "works importing array of hashes" do
+          scope.import [{ column => val1 }, { column => val2 }]
+
+          assert_equal [val1, val2], scope.map(&column).sort
+        end
+      end
+
+      it "works with a non-standard association primary key" do
+        user = User.create(id: 1, name: 'Solomon')
+        user.user_tokens.import [:id, :token], [[5, '12345abcdef67890']]
+
+        token = UserToken.find(5)
+        assert_equal 'Solomon', token.user_name
       end
     end
   end
 
+  context "importing model with polymorphic belongs_to" do
+    it "works without error" do
+      book     = FactoryBot.create :book
+      discount = Discount.new(discountable: book)
+
+      Discount.import([discount])
+
+      assert_equal 1, Discount.count
+    end
+  end
+
   context 'When importing models with Enum fields' do
     it 'should be able to import enum fields' do
       Book.delete_all if Book.count > 0
@@ -760,5 +880,19 @@ describe "#import" do
         end
       end
     end
+
+    context "with objects that respond to .to_sql as values" do
+      let(:columns) { %w(title author_name) }
+      let(:valid_values) { [["LDAP", Book.select("'Jerry Carter'").limit(1)], ["Rails Recipes", Book.select("'Chad Fowler'").limit(1)]] }
+
+      it "should import data" do
+        assert_difference "Topic.count", +2 do
+          Topic.import! columns, valid_values
+          topics = Topic.all
+          assert_equal "Jerry Carter", topics.first.author_name
+          assert_equal "Chad Fowler", topics.last.author_name
+        end
+      end
+    end
   end
 end

data/test/makara_postgis/import_test.rb

@@ -0,0 +1,8 @@
+require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
+require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples')
+
+should_support_postgresql_import_functionality
+
+if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
+  should_support_postgresql_upsert_functionality
+end

data/test/models/account.rb

@@ -0,0 +1,3 @@
+class Account < ActiveRecord::Base
+  self.locking_column = :lock
+end

data/test/models/bike_maker.rb

@@ -0,0 +1,7 @@
+module Bike
+  def self.table_name_prefix
+    'bike_'
+  end
+  class Maker < ActiveRecord::Base
+  end
+end

data/test/models/topic.rb

@@ -2,6 +2,8 @@ class Topic < ActiveRecord::Base
   validates_presence_of :author_name
   validates :title, numericality: { only_integer: true }, on: :context_test
   validates :title, uniqueness: true
+  validates :content, uniqueness: true
+  validates :word_count, numericality: { greater_than: 0 }, if: :content?
 
   validate -> { errors.add(:title, :validate_failed) if title == 'validate_failed' }
   before_validation -> { errors.add(:title, :invalid) if title == 'invalid' }
@@ -10,4 +12,12 @@ class Topic < ActiveRecord::Base
   belongs_to :parent, class_name: "Topic"
 
   composed_of :description, mapping: [%w(title title), %w(author_name author_name)], allow_nil: true, class_name: "TopicDescription"
+
+  default_scope { where(approved: true) }
+
+  private
+
+  def word_count
+    @word_count ||= content.to_s.scan(/\w+/).count
+  end
 end

data/test/models/user.rb

@@ -0,0 +1,3 @@
+class User < ActiveRecord::Base
+  has_many :user_tokens, primary_key: :name, foreign_key: :user_name
+end

data/test/models/user_token.rb

@@ -0,0 +1,4 @@
+class UserToken < ActiveRecord::Base
+  belongs_to :user, primary_key: :name, foreign_key: :user_name
+  validates :user, presence: true
+end

data/test/schema/generic_schema.rb

@@ -159,6 +159,26 @@ ActiveRecord::Schema.define do
     t.string :Features
   end
 
+  create_table :users, force: :cascade do |t|
+    t.string :name, null: false
+    t.integer :lock_version, null: false, default: 0
+  end
+
+  create_table :user_tokens, force: :cascade do |t|
+    t.string :user_name, null: false
+    t.string :token, null: false
+  end
+
+  create_table :accounts, force: :cascade do |t|
+    t.string :name, null: false
+    t.integer :lock, null: false, default: 0
+  end
+
+  create_table :bike_makers, force: :cascade do |t|
+    t.string :name, null: false
+    t.integer :lock_version, null: false, default: 0
+  end
+
   add_index :cars, :Name, unique: true
 
   unless ENV["SKIP_COMPOSITE_PK"]

data/test/schema/mysql2_schema.rb

@@ -0,0 +1,19 @@
+ActiveRecord::Schema.define do
+  create_table :books, force: :cascade do |t|
+    t.string :title, null: false
+    t.virtual :upper_title, type: :string, as: "upper(`title`)" if t.respond_to?(:virtual)
+    t.string :publisher, null: false, default: 'Default Publisher'
+    t.string :author_name, null: false
+    t.datetime :created_at
+    t.datetime :created_on
+    t.datetime :updated_at
+    t.datetime :updated_on
+    t.date :publish_date
+    t.integer :topic_id
+    t.integer :tag_id
+    t.integer :publisher_id
+    t.boolean :for_sale, default: true
+    t.integer :status, default: 0
+    t.string :type
+  end
+end

data/test/schema/postgresql_schema.rb

@@ -36,6 +36,7 @@ ActiveRecord::Schema.define do
     t.column :alarm_type, :integer, null: false
     t.column :status, :integer, null: false
     t.column :metadata, :text
+    t.column :secret_key, :binary
     t.datetime :created_at
     t.datetime :updated_at
   end

data/test/schema/sqlite3_schema.rb

@@ -0,0 +1,13 @@
+ActiveRecord::Schema.define do
+  create_table :alarms, force: true do |t|
+    t.column :device_id, :integer, null: false
+    t.column :alarm_type, :integer, null: false
+    t.column :status, :integer, null: false
+    t.column :metadata, :text
+    t.column :secret_key, :binary
+    t.datetime :created_at
+    t.datetime :updated_at
+  end
+
+  add_index :alarms, [:device_id, :alarm_type], unique: true, where: 'status <> 0'
+end

data/test/support/factories.rb

@@ -1,4 +1,4 @@
-FactoryGirl.define do
+FactoryBot.define do
   sequence(:book_title) { |n| "Book #{n}" }
   sequence(:chapter_title) { |n| "Chapter #{n}" }
   sequence(:end_note) { |n| "Endnote #{n}" }
@@ -9,12 +9,13 @@ FactoryGirl.define do
 
   factory :invalid_topic, class: "Topic" do
     sequence(:title) { |n| "Title #{n}" }
-    author_name nil
+    author_name { nil }
   end
 
   factory :topic do
     sequence(:title) { |n| "Title #{n}" }
     sequence(:author_name) { |n| "Author #{n}" }
+    sequence(:content) { |n| "Content #{n}" }
   end
 
   factory :widget do
@@ -26,7 +27,7 @@ FactoryGirl.define do
 
     trait :with_rule do
       after(:build) do |question|
-        question.build_rule(FactoryGirl.attributes_for(:rule))
+        question.build_rule(FactoryBot.attributes_for(:rule))
       end
     end
   end
@@ -39,21 +40,21 @@ FactoryGirl.define do
   factory :topic_with_book, parent: :topic do
     after(:build) do |topic|
       2.times do
-        book = topic.books.build(title: FactoryGirl.generate(:book_title), author_name: 'Stephen King')
+        book = topic.books.build(title: FactoryBot.generate(:book_title), author_name: 'Stephen King')
         3.times do
-          book.chapters.build(title: FactoryGirl.generate(:chapter_title))
+          book.chapters.build(title: FactoryBot.generate(:chapter_title))
         end
 
         4.times do
-          book.end_notes.build(note: FactoryGirl.generate(:end_note))
+          book.end_notes.build(note: FactoryBot.generate(:end_note))
         end
       end
     end
   end
 
   factory :book do
-    title 'Tortilla Flat'
-    author_name 'John Steinbeck'
+    title { 'Tortilla Flat' }
+    author_name { 'John Steinbeck' }
   end
 
   factory :car do

data/test/support/generate.rb

@@ -2,28 +2,28 @@ class ActiveSupport::TestCase
   def Build(*args) # rubocop:disable Style/MethodName
     n = args.shift if args.first.is_a?(Numeric)
     factory = args.shift
-    factory_girl_args = args.shift || {}
+    factory_bot_args = args.shift || {}
 
     if n
       [].tap do |collection|
-        n.times.each { collection << FactoryGirl.build(factory.to_s.singularize.to_sym, factory_girl_args) }
+        n.times.each { collection << FactoryBot.build(factory.to_s.singularize.to_sym, factory_bot_args) }
       end
     else
-      FactoryGirl.build(factory.to_s.singularize.to_sym, factory_girl_args)
+      FactoryBot.build(factory.to_s.singularize.to_sym, factory_bot_args)
     end
   end
 
   def Generate(*args) # rubocop:disable Style/MethodName
     n = args.shift if args.first.is_a?(Numeric)
     factory = args.shift
-    factory_girl_args = args.shift || {}
+    factory_bot_args = args.shift || {}
 
     if n
       [].tap do |collection|
-        n.times.each { collection << FactoryGirl.create(factory.to_s.singularize.to_sym, factory_girl_args) }
+        n.times.each { collection << FactoryBot.create(factory.to_s.singularize.to_sym, factory_bot_args) }
       end
     else
-      FactoryGirl.create(factory.to_s.singularize.to_sym, factory_girl_args)
+      FactoryBot.create(factory.to_s.singularize.to_sym, factory_bot_args)
     end
   end
 end

data/test/support/mysql/import_examples.rb

@@ -64,8 +64,8 @@ def should_support_mysql_import_functionality
       let(:columns) { %w(id author_name title) }
 
       setup do
-        topics << Topic.create!(title: "LDAP", author_name: "Big Bird")
-        topics << Topic.create!(title: "Rails Recipes", author_name: "Elmo")
+        topics << Topic.create!(title: "LDAP", author_name: "Big Bird", content: "Putting Directories to Work.")
+        topics << Topic.create!(title: "Rails Recipes", author_name: "Elmo", content: "A trusted collection of solutions.")
       end
 
       it "synchronizes passed in ActiveRecord model instances with the data just imported" do
@@ -82,5 +82,17 @@ def should_support_mysql_import_functionality
         assert_equal "Chad Fowler", topics.last.author_name, "wrong author!"
       end
     end
+
+    if ENV['AR_VERSION'].to_f >= 5.1
+      context "with virtual columns" do
+        let(:books) { [Book.new(author_name: "foo", title: "bar")] }
+
+        it "ignores virtual columns and creates record" do
+          assert_difference "Book.count", +1 do
+            Book.import books
+          end
+        end
+      end
+    end
   end
 end