activerecord-import 1.0.1 → 1.0.7

data/activerecord-import.gemspec

@@ -6,8 +6,8 @@ Gem::Specification.new do |gem|
   gem.email         = ["zach.dennis@gmail.com"]
   gem.summary       = "Bulk insert extension for ActiveRecord"
   gem.description   = "A library for bulk inserting data using ActiveRecord."
-  gem.homepage      = "http://github.com/zdennis/activerecord-import"
-  gem.license       = "Ruby"
+  gem.homepage      = "https://github.com/zdennis/activerecord-import"
+  gem.license       = "MIT"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = ActiveRecord::Import::VERSION
 
-  gem.required_ruby_version = ">= 1.9.2"
+  gem.required_ruby_version = ">= 2.0.0"
 
   gem.add_runtime_dependency "activerecord", ">= 3.2"
   gem.add_development_dependency "rake"

data/gemfiles/6.0.gemfile

@@ -0,0 +1 @@
+gem 'activerecord', '~> 6.0.0'

data/gemfiles/6.1.gemfile

@@ -0,0 +1 @@
+gem 'activerecord', '~> 6.1.0.alpha', github: "rails/rails"

data/lib/activerecord-import/adapters/abstract_adapter.rb

@@ -46,7 +46,7 @@ module ActiveRecord::Import::AbstractAdapter
 
       if supports_on_duplicate_key_update? && options[:on_duplicate_key_update]
         post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update], options[:primary_key], options[:locking_column] )
-      elsif options[:on_duplicate_key_update]
+      elsif logger && options[:on_duplicate_key_update]
         logger.warn "Ignoring on_duplicate_key_update because it is not supported by the database."
       end
 
@@ -59,6 +59,12 @@ module ActiveRecord::Import::AbstractAdapter
       post_sql_statements
     end
 
+    def increment_locking_column!(table_name, results, locking_column)
+      if locking_column.present?
+        results << "\"#{locking_column}\"=#{table_name}.\"#{locking_column}\"+1"
+      end
+    end
+
     def supports_on_duplicate_key_update?
       false
     end

data/lib/activerecord-import/adapters/mysql_adapter.rb

@@ -56,9 +56,9 @@ module ActiveRecord::Import::MysqlAdapter
   # in a single packet
   def max_allowed_packet # :nodoc:
     @max_allowed_packet ||= begin
-      result = execute( "SHOW VARIABLES like 'max_allowed_packet'" )
+      result = execute( "SELECT @@max_allowed_packet" )
       # original Mysql gem responds to #fetch_row while Mysql2 responds to #first
-      val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1]
+      val = result.respond_to?(:fetch_row) ? result.fetch_row[0] : result.first[0]
       val.to_i
     end
   end
@@ -102,7 +102,7 @@ module ActiveRecord::Import::MysqlAdapter
       qc = quote_column_name( column )
       "#{table_name}.#{qc}=VALUES(#{qc})"
     end
-    increment_locking_column!(results, table_name, locking_column)
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
@@ -112,7 +112,7 @@ module ActiveRecord::Import::MysqlAdapter
       qc2 = quote_column_name( column2 )
       "#{table_name}.#{qc1}=VALUES( #{qc2} )"
     end
-    increment_locking_column!(results, table_name, locking_column)
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',')
   end
 
@@ -121,9 +121,9 @@ module ActiveRecord::Import::MysqlAdapter
     exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry')
   end
 
-  def increment_locking_column!(results, table_name, locking_column)
+  def increment_locking_column!(table_name, results, locking_column)
     if locking_column.present?
-      results << "#{table_name}.`#{locking_column}`=`#{locking_column}`+1"
+      results << "`#{locking_column}`=#{table_name}.`#{locking_column}`+1"
     end
   end
 end

data/lib/activerecord-import/adapters/postgresql_adapter.rb

@@ -73,7 +73,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
       if (options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update] && !options[:recursive]
         sql << sql_for_on_duplicate_key_ignore( table_name, options[:on_duplicate_key_ignore] )
       end
-    elsif options[:on_duplicate_key_ignore] && !options[:on_duplicate_key_update]
+    elsif logger && options[:on_duplicate_key_ignore] && !options[:on_duplicate_key_update]
       logger.warn "Ignoring on_duplicate_key_ignore because it is not supported by the database."
     end
 
@@ -158,7 +158,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
       qc = quote_column_name( column )
       "#{qc}=EXCLUDED.#{qc}"
     end
-    increment_locking_column!(results, locking_column)
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
@@ -168,7 +168,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
       qc2 = quote_column_name( column2 )
       "#{qc1}=EXCLUDED.#{qc2}"
     end
-    increment_locking_column!(results, locking_column)
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
@@ -195,17 +195,17 @@ module ActiveRecord::Import::PostgreSQLAdapter
     exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
   end
 
-  def supports_on_duplicate_key_update?(current_version = postgresql_version)
-    current_version >= MIN_VERSION_FOR_UPSERT
+  def supports_on_duplicate_key_update?
+    database_version >= MIN_VERSION_FOR_UPSERT
   end
 
   def supports_setting_primary_key_of_imported_objects?
     true
   end
 
-  def increment_locking_column!(results, locking_column)
-    if locking_column.present?
-      results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
-    end
+  private
+
+  def database_version
+    defined?(postgresql_version) ? postgresql_version : super
   end
 end

data/lib/activerecord-import/adapters/sqlite3_adapter.rb

@@ -9,16 +9,12 @@ module ActiveRecord::Import::SQLite3Adapter
   # Override our conformance to ActiveRecord::Import::ImportSupport interface
   # to ensure that we only support import in supported version of SQLite.
   # Which INSERT statements with multiple value sets was introduced in 3.7.11.
-  def supports_import?(current_version = sqlite_version)
-    if current_version >= MIN_VERSION_FOR_IMPORT
-      true
-    else
-      false
-    end
+  def supports_import?
+    database_version >= MIN_VERSION_FOR_IMPORT
   end
 
-  def supports_on_duplicate_key_update?(current_version = sqlite_version)
-    current_version >= MIN_VERSION_FOR_UPSERT
+  def supports_on_duplicate_key_update?
+    database_version >= MIN_VERSION_FOR_UPSERT
   end
 
   # +sql+ can be a single string or an array. If it is an array all
@@ -96,7 +92,7 @@ module ActiveRecord::Import::SQLite3Adapter
 
   # Returns a generated ON CONFLICT DO UPDATE statement given the passed
   # in +args+.
-  def sql_for_on_duplicate_key_update( _table_name, *args ) # :nodoc:
+  def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
     arg, primary_key, locking_column = args
     arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
     return unless arg.is_a?( Hash )
@@ -117,9 +113,9 @@ module ActiveRecord::Import::SQLite3Adapter
 
     sql << "#{conflict_target}DO UPDATE SET "
     if columns.is_a?( Array )
-      sql << sql_for_on_duplicate_key_update_as_array( locking_column, columns )
+      sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, columns )
     elsif columns.is_a?( Hash )
-      sql << sql_for_on_duplicate_key_update_as_hash( locking_column, columns )
+      sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, columns )
     elsif columns.is_a?( String )
       sql << columns
     else
@@ -131,22 +127,22 @@ module ActiveRecord::Import::SQLite3Adapter
     sql
   end
 
-  def sql_for_on_duplicate_key_update_as_array( locking_column, arr ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
     results = arr.map do |column|
       qc = quote_column_name( column )
       "#{qc}=EXCLUDED.#{qc}"
     end
-    increment_locking_column!(results, locking_column)
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
-  def sql_for_on_duplicate_key_update_as_hash( locking_column, hsh ) # :nodoc:
+  def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
     results = hsh.map do |column1, column2|
       qc1 = quote_column_name( column1 )
       qc2 = quote_column_name( column2 )
       "#{qc1}=EXCLUDED.#{qc2}"
     end
-    increment_locking_column!(results, locking_column)
+    increment_locking_column!(table_name, results, locking_column)
     results.join( ',' )
   end
 
@@ -170,9 +166,9 @@ module ActiveRecord::Import::SQLite3Adapter
     exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
   end
 
-  def increment_locking_column!(results, locking_column)
-    if locking_column.present?
-      results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
-    end
+  private
+
+  def database_version
+    defined?(sqlite_version) ? sqlite_version : super
   end
 end

data/lib/activerecord-import/base.rb

@@ -13,6 +13,7 @@ module ActiveRecord::Import
     when 'postgresql_makara' then 'postgresql'
     when 'makara_postgis' then 'postgresql'
     when 'postgis' then 'postgresql'
+    when 'cockroachdb' then 'postgresql'
     else adapter
     end
   end
@@ -26,7 +27,13 @@ module ActiveRecord::Import
 
   # Loads the import functionality for the passed in ActiveRecord connection
   def self.load_from_connection_pool(connection_pool)
-    require_adapter connection_pool.spec.config[:adapter]
+    adapter =
+      if connection_pool.respond_to?(:db_config) # ActiveRecord >= 6.1
+        connection_pool.db_config.adapter
+      else
+        connection_pool.spec.config[:adapter]
+      end
+    require_adapter adapter
   end
 end
 

data/lib/activerecord-import/import.rb

@@ -26,6 +26,7 @@ module ActiveRecord::Import #:nodoc:
   class Validator
     def initialize(klass, options = {})
       @options = options
+      @validator_class = klass
       init_validations(klass)
     end
 
@@ -70,6 +71,8 @@ module ActiveRecord::Import #:nodoc:
     end
 
     def valid_model?(model)
+      init_validations(model.class) unless model.class == @validator_class
+
       validation_context = @options[:validate_with_context]
       validation_context ||= (model.new_record? ? :create : :update)
       current_context = model.send(:validation_context)
@@ -242,16 +245,17 @@ class ActiveRecord::Associations::CollectionAssociation
   alias import bulk_import unless respond_to? :import
 end
 
+module ActiveRecord::Import::Connection
+  def establish_connection(args = nil)
+    conn = super(args)
+    ActiveRecord::Import.load_from_connection_pool connection_pool
+    conn
+  end
+end
+
 class ActiveRecord::Base
   class << self
-    def establish_connection_with_activerecord_import(*args)
-      conn = establish_connection_without_activerecord_import(*args)
-      ActiveRecord::Import.load_from_connection_pool connection_pool
-      conn
-    end
-
-    alias establish_connection_without_activerecord_import establish_connection
-    alias establish_connection establish_connection_with_activerecord_import
+    prepend ActiveRecord::Import::Connection
 
     # Returns true if the current database connection adapter
     # supports import functionality, otherwise returns false.
@@ -543,7 +547,7 @@ class ActiveRecord::Base
     alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
 
     def import_helper( *args )
-      options = { validate: true, timestamps: true }
+      options = { validate: true, timestamps: true, track_validation_failures: false }
       options.merge!( args.pop ) if args.last.is_a? Hash
       # making sure that current model's primary key is used
       options[:primary_key] = primary_key
@@ -578,7 +582,7 @@ class ActiveRecord::Base
           if respond_to?(:timestamp_attributes_for_update, true)
             send(:timestamp_attributes_for_update).map(&:to_sym)
           else
-            new.send(:timestamp_attributes_for_update_in_model)
+            allocate.send(:timestamp_attributes_for_update_in_model)
           end
         end
 
@@ -627,7 +631,7 @@ class ActiveRecord::Base
         end
         # supports empty array
       elsif args.last.is_a?( Array ) && args.last.empty?
-        return ActiveRecord::Import::Result.new([], 0, [])
+        return ActiveRecord::Import::Result.new([], 0, [], [])
         # supports 2-element array and array
       elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
 
@@ -699,14 +703,18 @@ class ActiveRecord::Base
             # keep track of the instance and the position it is currently at. if this fails
             # validation we'll use the index to remove it from the array_of_attributes
             arr.each_with_index do |hsh, i|
-              model = new
-              hsh.each_pair { |k, v| model[k] = v }
+              # utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
+              model = new do |m|
+                hsh.each_pair { |k, v| m[k] = v }
+              end
+
               next if validator.valid_model?(model)
               raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
+
               array_of_attributes[i] = nil
               failure = model.dup
               failure.errors.send(:initialize_dup, model.errors)
-              failed_instances << failure
+              failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
             end
             array_of_attributes.compact!
           end
@@ -804,7 +812,7 @@ class ActiveRecord::Base
           result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
             batch_values,
             options,
-            "#{model_name} Create Many Without Validations Or Callbacks" )
+            "#{model_name} Create Many" )
           number_inserted += result.num_inserts
           ids += result.ids
           results += result.results
@@ -838,6 +846,19 @@ class ActiveRecord::Base
         end
       end
 
+      deserialize_value = lambda do |column, value|
+        column = columns_hash[column]
+        return value unless column
+        if respond_to?(:type_caster)
+          type = type_for_attribute(column.name)
+          type.deserialize(value)
+        elsif column.respond_to?(:type_cast_from_database)
+          column.type_cast_from_database(value)
+        else
+          value
+        end
+      end
+
       if models.size == import_result.results.size
         columns = Array(options[:returning])
         single_column = "#{columns.first}=" if columns.size == 1
@@ -845,10 +866,12 @@ class ActiveRecord::Base
           model = models[index]
 
           if single_column
-            model.send(single_column, result)
+            val = deserialize_value.call(columns.first, result)
+            model.send(single_column, val)
           else
             columns.each_with_index do |column, col_index|
-              model.send("#{column}=", result[col_index])
+              val = deserialize_value.call(column, result[col_index])
+              model.send("#{column}=", val)
             end
           end
         end
@@ -869,10 +892,12 @@ class ActiveRecord::Base
 
     # Sync belongs_to association ids with foreign key field
     def load_association_ids(model)
+      changed_columns = model.changed
       association_reflections = model.class.reflect_on_all_associations(:belongs_to)
       association_reflections.each do |association_reflection|
         column_name = association_reflection.foreign_key
         next if association_reflection.options[:polymorphic]
+        next if changed_columns.include?(column_name)
         association = model.association(association_reflection.name)
         association = association.target
         next if association.blank? || model.public_send(column_name).present?
@@ -952,7 +977,7 @@ class ActiveRecord::Base
           elsif column
             if respond_to?(:type_caster)                                         # Rails 5.0 and higher
               type = type_for_attribute(column.name)
-              val = type.type == :boolean ? type.cast(val) : type.serialize(val)
+              val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
               connection_memo.quote(val)
             elsif column.respond_to?(:type_cast_from_user)                       # Rails 4.2
               connection_memo.quote(column.type_cast_from_user(val), column)
@@ -961,7 +986,7 @@ class ActiveRecord::Base
                 val = serialized_attributes[column.name].dump(val)
               end
               # Fixes #443 to support binary (i.e. bytea) columns on PG
-              val = column.type_cast(val) unless column.type.to_sym == :binary
+              val = column.type_cast(val) unless column.type && column.type.to_sym == :binary
               connection_memo.quote(val, column)
             end
           else
@@ -980,7 +1005,7 @@ class ActiveRecord::Base
         timestamp_columns[:create] = timestamp_attributes_for_create_in_model
         timestamp_columns[:update] = timestamp_attributes_for_update_in_model
       else
-        instance = new
+        instance = allocate
         timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
         timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
       end

data/lib/activerecord-import/synchronize.rb

@@ -39,8 +39,8 @@ module ActiveRecord # :nodoc:
 
         next unless matched_instance
 
-        instance.send :clear_aggregation_cache
         instance.send :clear_association_cache
+        instance.send :clear_aggregation_cache if instance.respond_to?(:clear_aggregation_cache, true)
         instance.instance_variable_set :@attributes, matched_instance.instance_variable_get(:@attributes)
 
         if instance.respond_to?(:clear_changes_information)

data/lib/activerecord-import/version.rb

@@ -1,5 +1,5 @@
 module ActiveRecord
   module Import
-    VERSION = "1.0.1".freeze
+    VERSION = "1.0.7".freeze
   end
 end

data/test/import_test.rb

@@ -252,6 +252,16 @@ describe "#import" do
         end
       end
 
+      it "should index the failed instances by their poistion in the set if `track_failures` is true" do
+        index_offset = valid_values.length
+        results = Topic.import columns, valid_values + invalid_values, validate: true, track_validation_failures: true
+        assert_equal invalid_values.size, results.failed_instances.size
+        invalid_values.each_with_index do |value_set, index|
+          assert_equal index + index_offset, results.failed_instances[index].first
+          assert_equal value_set.first, results.failed_instances[index].last.title
+        end
+      end
+
       it "should set ids in valid models if adapter supports setting primary key of imported objects" do
         if ActiveRecord::Base.supports_setting_primary_key_of_imported_objects?
           Topic.import (invalid_models + valid_models), validate: true
@@ -900,4 +910,33 @@ describe "#import" do
       end
     end
   end
+  describe "importing model with after_initialize callback" do
+    let(:columns) { %w(name size) }
+    let(:valid_values) { [%w("Deer", "Small"), %w("Monkey", "Medium")] }
+    let(:invalid_values) do
+      [
+        { name: "giraffe", size: "Large" },
+        { size: "Medium" } # name is missing
+      ]
+    end
+    context "with validation checks turned off" do
+      it "should import valid data" do
+        Animal.import(columns, valid_values, validate: false)
+        assert_equal 2, Animal.count
+      end
+      it "should raise ArgumentError" do
+        assert_raise(ArgumentError) { Animal.import(invalid_values, validate: false) }
+      end
+    end
+
+    context "with validation checks turned on" do
+      it "should import valid data" do
+        Animal.import(columns, valid_values, validate: true)
+        assert_equal 2, Animal.count
+      end
+      it "should raise ArgumentError" do
+        assert_raise(ArgumentError) { Animal.import(invalid_values, validate: true) }
+      end
+    end
+  end
 end