activerecord-import 0.17.2 → 1.1.0

data/lib/activerecord-import/synchronize.rb

@@ -31,7 +31,7 @@ module ActiveRecord # :nodoc:
 
       klass = instances.first.class
 
-      fresh_instances = klass.where(conditions).order(order)
+      fresh_instances = klass.unscoped.where(conditions).order(order)
       instances.each do |instance|
         matched_instance = fresh_instances.detect do |fresh_instance|
           keys.all? { |key| fresh_instance.send(key) == instance.send(key) }
@@ -39,8 +39,8 @@ module ActiveRecord # :nodoc:
 
         next unless matched_instance
 
-        instance.send :clear_aggregation_cache
         instance.send :clear_association_cache
+        instance.send :clear_aggregation_cache if instance.respond_to?(:clear_aggregation_cache, true)
         instance.instance_variable_set :@attributes, matched_instance.instance_variable_get(:@attributes)
 
         if instance.respond_to?(:clear_changes_information)

data/lib/activerecord-import/value_sets_parser.rb

@@ -1,4 +1,14 @@
+require 'active_support/core_ext/array'
+
 module ActiveRecord::Import
+  class ValueSetTooLargeError < StandardError
+    attr_reader :size
+    def initialize(msg = "Value set exceeds max size", size = 0)
+      @size = size
+      super(msg)
+    end
+  end
+
   class ValueSetsBytesParser
     attr_reader :reserved_bytes, :max_bytes, :values
 
@@ -18,6 +28,12 @@ module ActiveRecord::Import
       current_size = 0
       values.each_with_index do |val, i|
         comma_bytes = arr.size
+        insert_size = reserved_bytes + val.bytesize
+
+        if insert_size > max_bytes
+          raise ValueSetTooLargeError.new("#{insert_size} bytes exceeds the max allowed for an insert [#{@max_bytes}]", insert_size)
+        end
+
         bytes_thus_far = reserved_bytes + current_size + val.bytesize + comma_bytes
         if bytes_thus_far <= max_bytes
           current_size += val.bytesize

data/lib/activerecord-import/version.rb

@@ -1,5 +1,5 @@
 module ActiveRecord
   module Import
-    VERSION = "0.17.2".freeze
+    VERSION = "1.1.0".freeze
   end
 end

data/test/adapters/makara_postgis.rb

@@ -0,0 +1 @@
+ENV["ARE_DB"] = "postgis"

data/test/import_test.rb

@@ -17,6 +17,11 @@ describe "#import" do
     assert_equal error.message, "Last argument should be a two dimensional array '[[]]'. First element in array was a String"
   end
 
+  it "warns you that you're passing more data than you ought to" do
+    error = assert_raise(ArgumentError) { Topic.import %w(title author_name), [['Author #1', 'Book #1', 0]] }
+    assert_equal error.message, "Number of values (8) exceeds number of columns (7)"
+  end
+
   it "should not produce an error when importing empty arrays" do
     assert_nothing_raised do
       Topic.import []
@@ -86,23 +91,73 @@ describe "#import" do
         assert_nil t.author_email_address
       end
     end
-  end
 
-  describe "with composite primary keys" do
-    it "should import models successfully" do
-      tags = [Tag.new(tag_id: 1, publisher_id: 1, tag: 'Mystery')]
+    context "with extra keys" do
+      let(:values) do
+        [
+          { title: "LDAP", author_name: "Jerry Carter" },
+          { title: "Rails Recipes", author_name: "Chad Fowler", author_email_address: "cfowler@test.com" } # author_email_address is unknown
+        ]
+      end
+
+      it "should fail when column names are not specified" do
+        err = assert_raises ArgumentError do
+          Topic.import values, validate: false
+        end
+
+        assert err.message.include? 'Extra keys: [:author_email_address]'
+      end
+
+      it "should succeed when column names are specified" do
+        assert_difference "Topic.count", +2 do
+          Topic.import columns, values, validate: false
+        end
+      end
+    end
+
+    context "with missing keys" do
+      let(:values) do
+        [
+          { title: "LDAP", author_name: "Jerry Carter" },
+          { title: "Rails Recipes" } # author_name is missing
+        ]
+      end
+
+      it "should fail when column names are not specified" do
+        err = assert_raises ArgumentError do
+          Topic.import values, validate: false
+        end
+
+        assert err.message.include? 'Missing keys: [:author_name]'
+      end
+
+      it "should fail on missing hash key from specified column names" do
+        err = assert_raises ArgumentError do
+          Topic.import %i(author_name), values, validate: false
+        end
 
-      assert_difference "Tag.count", +1 do
-        Tag.import tags
+        assert err.message.include? 'Missing keys: [:author_name]'
       end
     end
+  end
+
+  unless ENV["SKIP_COMPOSITE_PK"]
+    describe "with composite primary keys" do
+      it "should import models successfully" do
+        tags = [Tag.new(tag_id: 1, publisher_id: 1, tag: 'Mystery')]
+
+        assert_difference "Tag.count", +1 do
+          Tag.import tags
+        end
+      end
 
-    it "should import array of values successfully" do
-      columns = [:tag_id, :publisher_id, :tag]
-      values = [[1, 1, 'Mystery'], [2, 1, 'Science']]
+      it "should import array of values successfully" do
+        columns = [:tag_id, :publisher_id, :tag]
+        values = [[1, 1, 'Mystery'], [2, 1, 'Science']]
 
-      assert_difference "Tag.count", +2 do
-        Tag.import columns, values, validate: false
+        assert_difference "Tag.count", +2 do
+          Tag.import columns, values, validate: false
+        end
       end
     end
   end
@@ -119,12 +174,12 @@ describe "#import" do
   end
 
   context "with :validation option" do
-    let(:columns) { %w(title author_name) }
-    let(:valid_values) { [["LDAP", "Jerry Carter"], ["Rails Recipes", "Chad Fowler"]] }
-    let(:valid_values_with_context) { [[1111, "Jerry Carter"], [2222, "Chad Fowler"]] }
-    let(:invalid_values) { [["The RSpec Book", ""], ["Agile+UX", ""]] }
-    let(:valid_models) { valid_values.map { |title, author_name| Topic.new(title: title, author_name: author_name) } }
-    let(:invalid_models) { invalid_values.map { |title, author_name| Topic.new(title: title, author_name: author_name) } }
+    let(:columns) { %w(title author_name content) }
+    let(:valid_values) { [["LDAP", "Jerry Carter", "Putting Directories to Work."], ["Rails Recipes", "Chad Fowler", "A trusted collection of solutions."]] }
+    let(:valid_values_with_context) { [[1111, "Jerry Carter", "1111"], [2222, "Chad Fowler", "2222"]] }
+    let(:invalid_values) { [["The RSpec Book", "David Chelimsky", "..."], ["Agile+UX", "", "All about Agile in UX."]] }
+    let(:valid_models) { valid_values.map { |title, author_name, content| Topic.new(title: title, author_name: author_name, content: content) } }
+    let(:invalid_models) { invalid_values.map { |title, author_name, content| Topic.new(title: title, author_name: author_name, content: content) } }
 
     context "with validation checks turned off" do
       it "should import valid data" do
@@ -159,6 +214,22 @@ describe "#import" do
         end
       end
 
+      it "should ignore uniqueness validators" do
+        Topic.import columns, valid_values
+        assert_difference "Topic.count", +2 do
+          Topic.import columns, valid_values
+        end
+      end
+
+      it "should not alter the callback chain of the model" do
+        attributes = columns.zip(valid_values.first).to_h
+        topic = Topic.new attributes
+        Topic.import [topic], validate: true
+        duplicate_topic = Topic.new attributes
+        Topic.import [duplicate_topic], validate: true
+        assert duplicate_topic.invalid?
+      end
+
       it "should not import invalid data" do
         assert_no_difference "Topic.count" do
           Topic.import columns, invalid_values, validate: true
@@ -181,8 +252,18 @@ describe "#import" do
         end
       end
 
+      it "should index the failed instances by their poistion in the set if `track_failures` is true" do
+        index_offset = valid_values.length
+        results = Topic.import columns, valid_values + invalid_values, validate: true, track_validation_failures: true
+        assert_equal invalid_values.size, results.failed_instances.size
+        invalid_values.each_with_index do |value_set, index|
+          assert_equal index + index_offset, results.failed_instances[index].first
+          assert_equal value_set.first, results.failed_instances[index].last.title
+        end
+      end
+
       it "should set ids in valid models if adapter supports setting primary key of imported objects" do
-        if ActiveRecord::Base.support_setting_primary_key_of_imported_objects?
+        if ActiveRecord::Base.supports_setting_primary_key_of_imported_objects?
           Topic.import (invalid_models + valid_models), validate: true
           assert_nil invalid_models[0].id
           assert_nil invalid_models[1].id
@@ -192,7 +273,7 @@ describe "#import" do
       end
 
       it "should set ActiveRecord timestamps in valid models if adapter supports setting primary key of imported objects" do
-        if ActiveRecord::Base.support_setting_primary_key_of_imported_objects?
+        if ActiveRecord::Base.supports_setting_primary_key_of_imported_objects?
           Timecop.freeze(Time.at(0)) do
             Topic.import (invalid_models + valid_models), validate: true
           end
@@ -215,6 +296,48 @@ describe "#import" do
         end
         assert_equal 0, Topic.where(title: invalid_values.map(&:first)).count
       end
+
+      it "should run callbacks" do
+        assert_no_difference "Topic.count" do
+          Topic.import columns, [["invalid", "Jerry Carter"]], validate: true
+        end
+      end
+
+      it "should call validation methods" do
+        assert_no_difference "Topic.count" do
+          Topic.import columns, [["validate_failed", "Jerry Carter"]], validate: true
+        end
+      end
+    end
+
+    context "with uniqueness validators included" do
+      it "should not import duplicate records" do
+        Topic.import columns, valid_values
+        assert_no_difference "Topic.count" do
+          Topic.import columns, valid_values, validate_uniqueness: true
+        end
+      end
+    end
+
+    context "when validatoring presence of belongs_to association" do
+      it "should not import records without foreign key" do
+        assert_no_difference "UserToken.count" do
+          UserToken.import [:token], [['12345abcdef67890']]
+        end
+      end
+
+      it "should import records with foreign key" do
+        assert_difference "UserToken.count", +1 do
+          UserToken.import [:user_name, :token], [%w("Bob", "12345abcdef67890")]
+        end
+      end
+
+      it "should not mutate the defined validations" do
+        UserToken.import [:user_name, :token], [%w("Bob", "12345abcdef67890")]
+        ut = UserToken.new
+        ut.valid?
+        assert_includes ut.errors.messages, :user
+      end
     end
   end
 
@@ -282,6 +405,15 @@ describe "#import" do
         assert_equal 3, result.num_inserts if Topic.supports_import?
       end
     end
+
+    it "should accept and call an optional callable to run after each batch" do
+      lambda_called = 0
+
+      my_proc = ->(_row_count, _batches, _batch, _duration) { lambda_called += 1 }
+      Topic.import Build(10, :topics), batch_size: 4, batch_progress: my_proc
+
+      assert_equal 3, lambda_called
+    end
   end
 
   context "with :synchronize option" do
@@ -290,7 +422,7 @@ describe "#import" do
 
       it "doesn't reload any data (doesn't work)" do
         Topic.import new_topics, synchronize: new_topics
-        if Topic.support_setting_primary_key_of_imported_objects?
+        if Topic.supports_setting_primary_key_of_imported_objects?
           assert new_topics.all?(&:persisted?), "Records should have been reloaded"
         else
           assert new_topics.all?(&:new_record?), "No record should have been reloaded"
@@ -412,11 +544,11 @@ describe "#import" do
 
     context "when the timestamps columns are present" do
       setup do
-        @existing_book = Book.create(title: "Fell", author_name: "Curry", publisher: "Bayer", created_at: 2.years.ago.utc, created_on: 2.years.ago.utc)
+        @existing_book = Book.create(title: "Fell", author_name: "Curry", publisher: "Bayer", created_at: 2.years.ago.utc, created_on: 2.years.ago.utc, updated_at: 2.years.ago.utc, updated_on: 2.years.ago.utc)
         ActiveRecord::Base.default_timezone = :utc
         Timecop.freeze(time) do
           assert_difference "Book.count", +2 do
-            Book.import %w(title author_name publisher created_at created_on), [["LDAP", "Big Bird", "Del Rey", nil, nil], [@existing_book.title, @existing_book.author_name, @existing_book.publisher, @existing_book.created_at, @existing_book.created_on]]
+            Book.import %w(title author_name publisher created_at created_on updated_at updated_on), [["LDAP", "Big Bird", "Del Rey", nil, nil, nil, nil], [@existing_book.title, @existing_book.author_name, @existing_book.publisher, @existing_book.created_at, @existing_book.created_on, @existing_book.updated_at, @existing_book.updated_on]]
           end
         end
         @new_book, @existing_book = Book.last 2
@@ -445,6 +577,23 @@ describe "#import" do
       it "should set the updated_on column for new records" do
         assert_in_delta time.to_i, @new_book.updated_on.to_i, 1.second
       end
+
+      it "should not set the updated_at column for existing records" do
+        assert_equal 2.years.ago.utc.strftime("%Y:%d"), @existing_book.updated_at.strftime("%Y:%d")
+      end
+
+      it "should not set the updated_on column for existing records" do
+        assert_equal 2.years.ago.utc.strftime("%Y:%d"), @existing_book.updated_on.strftime("%Y:%d")
+      end
+
+      it "should not set the updated_at column on models if changed" do
+        timestamp = Time.now.utc
+        books = [
+          Book.new(author_name: "Foo", title: "Baz", created_at: timestamp, updated_at: timestamp)
+        ]
+        Book.import books
+        assert_equal timestamp.strftime("%Y:%d"), Book.last.updated_at.strftime("%Y:%d")
+      end
     end
 
     context "when a custom time zone is set" do
@@ -489,7 +638,7 @@ describe "#import" do
 
   context "importing through an association scope" do
     { has_many: :chapters, polymorphic: :discounts }.each do |association_type, association|
-      book   = FactoryGirl.create :book
+      book   = FactoryBot.create :book
       scope  = book.public_send association
       klass  = { chapters: Chapter, discounts: Discount }[association]
       column = { chapters: :title,  discounts: :amount  }[association]
@@ -511,10 +660,35 @@ describe "#import" do
 
           assert_equal [val1, val2], scope.map(&column).sort
         end
+
+        it "works importing array of hashes" do
+          scope.import [{ column => val1 }, { column => val2 }]
+
+          assert_equal [val1, val2], scope.map(&column).sort
+        end
+      end
+
+      it "works with a non-standard association primary key" do
+        user = User.create(id: 1, name: 'Solomon')
+        user.user_tokens.import [:id, :token], [[5, '12345abcdef67890']]
+
+        token = UserToken.find(5)
+        assert_equal 'Solomon', token.user_name
       end
     end
   end
 
+  context "importing model with polymorphic belongs_to" do
+    it "works without error" do
+      book     = FactoryBot.create :book
+      discount = Discount.new(discountable: book)
+
+      Discount.import([discount])
+
+      assert_equal 1, Discount.count
+    end
+  end
+
   context 'When importing models with Enum fields' do
     it 'should be able to import enum fields' do
       Book.delete_all if Book.count > 0
@@ -589,6 +763,30 @@ describe "#import" do
     end
   end
 
+  context 'importing arrays of values with boolean fields' do
+    let(:columns) { [:author_name, :title, :for_sale] }
+
+    it 'should be able to coerce integers as boolean fields' do
+      Book.delete_all if Book.count > 0
+      values = [['Author #1', 'Book #1', 0], ['Author #2', 'Book #2', 1]]
+      assert_difference "Book.count", +2 do
+        Book.import columns, values
+      end
+      assert_equal false, Book.first.for_sale
+      assert_equal true, Book.last.for_sale
+    end
+
+    it 'should be able to coerce strings as boolean fields' do
+      Book.delete_all if Book.count > 0
+      values = [['Author #1', 'Book #1', 'false'], ['Author #2', 'Book #2', 'true']]
+      assert_difference "Book.count", +2 do
+        Book.import columns, values
+      end
+      assert_equal false, Book.first.for_sale
+      assert_equal true, Book.last.for_sale
+    end
+  end
+
   describe "importing when model has default_scope" do
     it "doesn't import the default scope values" do
       assert_difference "Widget.unscoped.count", +2 do
@@ -638,6 +836,16 @@ describe "#import" do
       assert_equal(data.as_json, Widget.find_by_w_id(9).json_data)
     end
 
+    it "imports serialized values from saved records" do
+      Widget.import [:w_id, :json_data], [[1, data]]
+      assert_equal data.as_json, Widget.last.json_data
+
+      w = Widget.last
+      w.w_id = 2
+      Widget.import([w])
+      assert_equal data.as_json, Widget.last.json_data
+    end
+
     context "with a store" do
       it "imports serialized attributes set using accessors" do
         vendors = [Vendor.new(name: 'Vendor 1', color: 'blue')]
@@ -696,5 +904,48 @@ describe "#import" do
         end
       end
     end
+
+    context "with objects that respond to .to_sql as values" do
+      let(:columns) { %w(title author_name) }
+      let(:valid_values) { [["LDAP", Book.select("'Jerry Carter'").limit(1)], ["Rails Recipes", Book.select("'Chad Fowler'").limit(1)]] }
+
+      it "should import data" do
+        assert_difference "Topic.count", +2 do
+          Topic.import! columns, valid_values
+          topics = Topic.all
+          assert_equal "Jerry Carter", topics.first.author_name
+          assert_equal "Chad Fowler", topics.last.author_name
+        end
+      end
+    end
+  end
+  describe "importing model with after_initialize callback" do
+    let(:columns) { %w(name size) }
+    let(:valid_values) { [%w("Deer", "Small"), %w("Monkey", "Medium")] }
+    let(:invalid_values) do
+      [
+        { name: "giraffe", size: "Large" },
+        { size: "Medium" } # name is missing
+      ]
+    end
+    context "with validation checks turned off" do
+      it "should import valid data" do
+        Animal.import(columns, valid_values, validate: false)
+        assert_equal 2, Animal.count
+      end
+      it "should raise ArgumentError" do
+        assert_raise(ArgumentError) { Animal.import(invalid_values, validate: false) }
+      end
+    end
+
+    context "with validation checks turned on" do
+      it "should import valid data" do
+        Animal.import(columns, valid_values, validate: true)
+        assert_equal 2, Animal.count
+      end
+      it "should raise ArgumentError" do
+        assert_raise(ArgumentError) { Animal.import(invalid_values, validate: true) }
+      end
+    end
   end
 end