activerecord-import 2.3.0 → 3.0.0

data/lib/activerecord-import/import.rb

@@ -23,126 +23,84 @@ module ActiveRecord::Import # :nodoc:
     end
   end
 
-  class Validator
-    def initialize(klass, options = {})
-      @options = options
-      @validator_class = klass
-      init_validations(klass)
+  # Skips uniqueness validations while a model is being validated for an import
+  # that doesn't validate uniqueness, including the validations Rails runs on
+  # the model's associated records.
+  module SkipUniquenessValidation # :nodoc:
+    def validate_each(record, attribute, value)
+      super unless ActiveRecord::Import::Validator.skip_uniqueness?
     end
+  end
 
-    def init_validations(klass)
-      @validate_callbacks = klass._validate_callbacks.dup
+  # Validates the presence of a belongs_to association on a model being
+  # validated for an import by checking its foreign key, so the associated
+  # record isn't loaded for every model.
+  module ForeignKeyPresenceValidation # :nodoc:
+    def validate(record)
+      validator = ActiveRecord::Import::Validator.foreign_key_presence_validator(self, record)
+      return super unless validator
 
-      @validate_callbacks.each_with_index do |callback, i|
-        filter = callback.respond_to?(:raw_filter) ? callback.raw_filter : callback.filter
-        next unless filter.class.name =~ /Validations::PresenceValidator/ ||
-                    (!@options[:validate_uniqueness] &&
-                     filter.is_a?(ActiveRecord::Validations::UniquenessValidator))
+      ActiveModel::EachValidator.instance_method(:validate).bind(validator).call(record)
+    end
+  end
 
-        callback = callback.dup
-        filter = filter.dup
-        attrs = filter.instance_variable_get(:@attributes).dup
+  class Validator
+    # Holds [validator, model] for the current thread while valid_model? runs.
+    CURRENT = :activerecord_import_validation
 
-        if filter.is_a?(ActiveRecord::Validations::UniquenessValidator)
-          attrs = []
-        else
-          associations = klass.reflect_on_all_associations(:belongs_to)
-          associations.each do |assoc|
-            if (index = attrs.index(assoc.name))
-              key = assoc.foreign_key.is_a?(Array) ? assoc.foreign_key.map(&:to_sym) : assoc.foreign_key.to_sym
-              attrs[index] = key unless attrs.include?(key)
-            end
-          end
-        end
+    attr_reader :options
 
-        filter.instance_variable_set(:@attributes, attrs.flatten)
+    def self.skip_uniqueness?
+      validator, = Thread.current[CURRENT]
+      validator ? !validator.options[:validate_uniqueness] : false
+    end
 
-        if @validate_callbacks.respond_to?(:chain, true)
-          @validate_callbacks.send(:chain).tap do |chain|
-            callback.instance_variable_set(:@filter, filter)
-            callback.instance_variable_set(:@compiled, nil)
-            chain[i] = callback
-          end
-        else
-          callback.raw_filter = filter
-          callback.filter = callback.send(:_compile_filter, filter)
-          @validate_callbacks[i] = callback
-        end
-      end
+    def self.foreign_key_presence_validator(presence_validator, record)
+      validator, model = Thread.current[CURRENT]
+      validator.foreign_key_presence_validator(presence_validator, record.class) if model.equal?(record)
     end
 
-    def valid_model?(model)
-      init_validations(model.class) unless model.instance_of?(@validator_class)
-
-      validation_context = @options[:validate_with_context]
-      validation_context ||= (model.new_record? ? :create : :update)
-      current_context = model.send(:validation_context)
-
-      begin
-        set_validation_context(model, validation_context)
-        model.errors.clear
-
-        model.run_callbacks(:validation) do
-          if defined?(ActiveSupport::Callbacks::Filters::Environment) # ActiveRecord >= 4.1
-            runner = if @validate_callbacks.method(:compile).arity == 0
-              @validate_callbacks.compile
-            else # ActiveRecord >= 7.1
-              @validate_callbacks.compile(nil)
-            end
-            env = ActiveSupport::Callbacks::Filters::Environment.new(model, false, nil)
-            if runner.respond_to?(:call) # ActiveRecord < 5.1
-              runner.call(env)
-            else # ActiveRecord >= 5.1
-              # Note that this is a gross simplification of ActiveSupport::Callbacks#run_callbacks.
-              # It's technically possible for there to exist an "around" callback in the
-              # :validate chain, but this would be an aberration, since Rails doesn't define
-              # "around_validate". Still, rather than silently ignoring such callbacks, we
-              # explicitly raise a RuntimeError, since activerecord-import was asked to perform
-              # validations and it's unable to do so.
-              #
-              # The alternative here would be to copy-and-paste the bulk of the
-              # ActiveSupport::Callbacks#run_callbacks method, which is undesirable if there's
-              # no real-world use case for it.
-              raise "The :validate callback chain contains an 'around' callback, which is unsupported" unless runner.final?
-              runner.invoke_before(env)
-              # Ensure a truthy value is returned. ActiveRecord < 7.2 always returned an array.
-              runner.invoke_after(env) || []
-            end
-          elsif @validate_callbacks.method(:compile).arity == 0 # ActiveRecord = 4.0
-            model.instance_eval @validate_callbacks.compile
-          else # ActiveRecord 3.x
-            model.instance_eval @validate_callbacks.compile(nil, model)
-          end
-        end
+    def initialize(options = {})
+      @options = options
+      @foreign_key_presence_validators = {}
+    end
 
-        model.errors.empty?
-      ensure
-        set_validation_context(model, current_context)
-      end
+    def valid_model?(model)
+      current = Thread.current[CURRENT]
+      Thread.current[CURRENT] = [self, model]
+      model.valid?(@options[:validate_with_context])
+    ensure
+      Thread.current[CURRENT] = current
     end
 
-    private
+    # Returns a copy of +presence_validator+ that validates the foreign keys of
+    # any belongs_to associations it validates, or nil if there are none.
+    def foreign_key_presence_validator(presence_validator, klass)
+      key = [presence_validator, klass]
+      return @foreign_key_presence_validators[key] if @foreign_key_presence_validators.key?(key)
+
+      attrs = presence_validator.attributes.dup
+      klass.reflect_on_all_associations(:belongs_to).each do |assoc|
+        if (index = attrs.index(assoc.name))
+          foreign_key = assoc.foreign_key.is_a?(Array) ? assoc.foreign_key.map(&:to_sym) : assoc.foreign_key.to_sym
+          attrs[index] = foreign_key unless attrs.include?(foreign_key)
+        end
+      end
 
-    # ActiveModel's private +validation_context=+ writer was removed in Rails
-    # 8.2 (rails/rails@0f9d1270834a6407a59637650bf910d8ae826169) in favor of a
-    # validation context object reachable through the private
-    # +context_for_validation+ reader. Support both so that validated imports
-    # keep working across all supported ActiveRecord versions.
-    def set_validation_context(model, context)
-      if model.respond_to?(:validation_context=, true)
-        model.send(:validation_context=, context)
-      else
-        model.send(:context_for_validation).context = context
+      @foreign_key_presence_validators[key] = if attrs != presence_validator.attributes
+        presence_validator.dup.tap { |validator| validator.instance_variable_set(:@attributes, attrs.flatten) }
       end
     end
   end
 end
 
+ActiveRecord::Validations::UniquenessValidator.prepend(ActiveRecord::Import::SkipUniquenessValidation)
+ActiveModel::Validations::PresenceValidator.prepend(ActiveRecord::Import::ForeignKeyPresenceValidation)
+
 class ActiveRecord::Associations::CollectionProxy
   def bulk_import(*args, &block)
     @association.bulk_import(*args, &block)
   end
-  alias import bulk_import unless respond_to? :import
 end
 
 class ActiveRecord::Associations::CollectionAssociation
@@ -154,9 +112,10 @@ class ActiveRecord::Associations::CollectionAssociation
     options = args.last.is_a?(Hash) ? args.pop : {}
 
     model_klass = reflection.klass
+    args = model_klass.send(:arrays_from_array_like_import_arguments, args)
     symbolized_foreign_key = reflection.foreign_key.to_sym
 
-    symbolized_column_names = if model_klass.connection_pool.with_connection { |conn| conn.respond_to?(:supports_virtual_columns?) && conn.supports_virtual_columns? }
+    symbolized_column_names = if model_klass.connection_pool.with_connection(&:supports_virtual_columns?)
       model_klass.columns.reject(&:virtual?).map { |c| c.name.to_sym }
     else
       model_klass.column_names.map(&:to_sym)
@@ -164,12 +123,14 @@ class ActiveRecord::Associations::CollectionAssociation
 
     owner_primary_key = reflection.active_record_primary_key.to_sym
     owner_primary_key_value = owner.send(owner_primary_key)
+    owner_polymorphic_name = owner.class.polymorphic_name if reflection.type
 
     # assume array of model objects
     if args.last.is_a?( Array ) && args.last.first.is_a?(ActiveRecord::Base)
       if args.length == 2
         models = args.last
         column_names = args.first.dup
+        symbolized_column_names = column_names.map(&:to_sym)
       else
         models = args.first
         column_names = symbolized_column_names
@@ -179,9 +140,13 @@ class ActiveRecord::Associations::CollectionAssociation
         column_names << symbolized_foreign_key
       end
 
+      if reflection.type && !symbolized_column_names.include?(reflection.type.to_sym)
+        column_names << reflection.type.to_sym
+      end
+
       models.each do |m|
         m.public_send "#{symbolized_foreign_key}=", owner_primary_key_value
-        m.public_send "#{reflection.type}=", owner.class.name if reflection.type
+        m.public_send "#{reflection.type}=", owner_polymorphic_name if reflection.type
       end
 
       model_klass.bulk_import column_names, models, options
@@ -198,6 +163,7 @@ class ActiveRecord::Associations::CollectionAssociation
         allow_extra_hash_keys = false
       end
 
+      required_column_names = column_names.dup
       symbolized_column_names = column_names.map(&:to_sym)
       unless symbolized_column_names.include?(symbolized_foreign_key)
         column_names << symbolized_foreign_key
@@ -208,15 +174,16 @@ class ActiveRecord::Associations::CollectionAssociation
       end
 
       array_of_attributes = array_of_hashes.map do |h|
-        error_message = model_klass.send(:validate_hash_import, h, symbolized_column_names, allow_extra_hash_keys)
+        error_message = model_klass.send(:validate_hash_import, h, required_column_names, allow_extra_hash_keys)
 
         raise ArgumentError, error_message if error_message
 
         column_names.map do |key|
-          if key == symbolized_foreign_key
+          symbolized_key = key.to_sym
+          if symbolized_key == symbolized_foreign_key
             owner_primary_key_value
-          elsif reflection.type && key == reflection.type.to_sym
-            owner.class.name
+          elsif reflection.type && symbolized_key == reflection.type.to_sym
+            owner_polymorphic_name
           else
             h[key]
           end
@@ -251,19 +218,18 @@ class ActiveRecord::Associations::CollectionAssociation
         symbolized_type = reflection.type.to_sym
         if symbolized_column_names.include?(symbolized_type)
           index = symbolized_column_names.index(symbolized_type)
-          array_of_attributes.each { |attrs| attrs[index] = owner.class.name }
+          array_of_attributes.each { |attrs| attrs[index] = owner_polymorphic_name }
         else
           column_names << symbolized_type
-          array_of_attributes.each { |attrs| attrs << owner.class.name }
+          array_of_attributes.each { |attrs| attrs << owner_polymorphic_name }
         end
       end
 
       model_klass.bulk_import column_names, array_of_attributes, options
     else
-      raise ArgumentError, "Invalid arguments!"
+      raise ArgumentError, model_klass.send(:invalid_import_arguments_message, args)
     end
   end
-  alias import bulk_import unless respond_to? :import
 end
 
 module ActiveRecord::Import::ConnectionHandler
@@ -383,7 +349,7 @@ class ActiveRecord::Base
     #   (if false) even if record timestamps is disabled in ActiveRecord::Base
     # * +recursive+ - true|false, tells import to import all has_many/has_one
     #   associations if the adapter supports setting the primary keys of the
-    #   newly imported objects. PostgreSQL only.
+    #   newly imported objects. PostgreSQL and MariaDB 10.5+ only.
     # * +batch_size+ - an integer value to specify the max number of records to
     #   include per insert. Defaults to the total number of records to import.
     #
@@ -438,7 +404,8 @@ class ActiveRecord::Base
     # ==== Using :all
     #
     # The :on_duplicate_key_update option can be set to :all. All columns
-    # other than the primary key are updated. If a list of column names is
+    # other than the primary key are updated, except for timestamps recorded
+    # on create, such as created_at. If a list of column names is
     # supplied, only those columns will be updated. Below is an example:
     #
     #   BlogPost.import columns, values, on_duplicate_key_update: :all
@@ -469,7 +436,8 @@ class ActiveRecord::Base
     # ==== Using :all
     #
     # The :on_duplicate_key_update option can be set to :all. All columns
-    # other than the primary key are updated. If a list of column names is
+    # other than the primary key are updated, except for timestamps recorded
+    # on create, such as created_at. If a list of column names is
     # supplied, only those columns will be updated. Below is an example:
     #
     #   BlogPost.import columns, values, on_duplicate_key_update: :all
@@ -534,7 +502,8 @@ class ActiveRecord::Base
     #
     # ===== Using :all
     #
-    # The :columns attribute can be :all. All columns other than the primary key will be updated.
+    # The :columns attribute can be :all. All columns other than the primary key will be updated,
+    # except for timestamps recorded on create, such as created_at.
     # If a list of column names is supplied, only those columns will be updated.
     # Below is an example:
     #
@@ -573,160 +542,58 @@ class ActiveRecord::Base
         import_helper(*args)
       end
     end
-    alias import bulk_import unless ActiveRecord::Base.respond_to? :import
 
     # Imports a collection of values if all values are valid. Import fails at the
     # first encountered validation error and raises ActiveRecord::RecordInvalid
     # with the failed instance.
     def bulk_import!(*args)
       options = args.last.is_a?( Hash ) ? args.pop : {}
-      options[:validate] = true
-      options[:raise_error] = true
+      options = options.merge( validate: true, raise_error: true )
 
       bulk_import(*args, options)
     end
-    alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
+
+    # Returns the column for +name+, following an attribute alias unless +name+ is
+    # itself a column, so a column that's also an alias isn't replaced by the
+    # column it aliases.
+    def column_name_for_import(name) # :nodoc:
+      name = name.to_s
+      attribute_alias?(name) && !columns_hash.key?(name) ? attribute_alias(name) : name
+    end
 
     def import_helper( *args )
       connection_pool.with_connection do |conn|
-        options = { model: self, validate: true, timestamps: true, track_validation_failures: false }
-        options.merge!( args.pop ) if args.last.is_a? Hash
-        # making sure that current model's primary key is used
-        options[:primary_key] = primary_key
-        options[:locking_column] = locking_column if locking_enabled?
+        # The adapter's import support is loaded when the connection is established, which is
+        # missed if activerecord-import is required afterwards, so load it now if it's missing.
+        ActiveRecord::Import.load_from_connection_pool_once(connection_pool) unless supports_import?(conn)
+
+        options, args = options_and_arguments_for_import(args)
 
         is_validating = options[:validate_with_context].present? || options[:validate]
-        validator = ActiveRecord::Import::Validator.new(self, options)
+        validator = ActiveRecord::Import::Validator.new(options) if is_validating
+
+        # supports empty array
+        return ActiveRecord::Import::Result.new([], 0, [], []) if args.last.is_a?( Array ) && args.last.empty?
 
         # assume array of model objects
         if args.last.is_a?( Array ) && args.last.first.is_a?(ActiveRecord::Base)
-          if args.length == 2
-            models = args.last
-            column_names = args.first.dup
-          else
-            models = args.first
-            column_names = if conn.try(:supports_virtual_columns?)
-              columns.reject(&:virtual?).map(&:name)
-            else
-              self.column_names.dup
-            end
-          end
-
-          if models.first.id.nil?
-            Array(primary_key).each do |c|
-              if column_names.include?(c) && schema_columns_hash(conn)[c].type == :uuid
-                column_names.delete(c)
-              end
-            end
-          end
-
-          update_attrs = if record_timestamps && options[:timestamps]
-            if respond_to?(:timestamp_attributes_for_update, true)
-              send(:timestamp_attributes_for_update).map(&:to_sym)
-            else
-              allocate.send(:timestamp_attributes_for_update_in_model)
-            end
-          end
-
-          array_of_attributes = []
-
-          models.each do |model|
-            if supports_setting_primary_key_of_imported_objects?(conn)
-              load_association_ids(model)
-            end
-
-            if is_validating && !validator.valid_model?(model)
-              raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
-              next
-            end
-
-            array_of_attributes << column_names.map do |name|
-              if model.persisted? &&
-                 update_attrs && update_attrs.include?(name.to_sym) &&
-                 !model.send("#{name}_changed?")
-                nil
-              else
-                model.read_attribute(name.to_s)
-              end
-            end
-          end
-          # supports array of hash objects
-        elsif args.last.is_a?( Array ) && args.last.first.is_a?(Hash)
-          if args.length == 2
-            array_of_hashes = args.last
-            column_names = args.first.dup
-            allow_extra_hash_keys = true
-          else
-            array_of_hashes = args.first
-            column_names = array_of_hashes.first.keys
-            allow_extra_hash_keys = false
-          end
-
-          array_of_attributes = array_of_hashes.map do |h|
-            error_message = validate_hash_import(h, column_names, allow_extra_hash_keys)
-
-            raise ArgumentError, error_message if error_message
-
-            column_names.map do |key|
-              h[key]
-            end
-          end
-          # supports empty array
-        elsif args.last.is_a?( Array ) && args.last.empty?
-          return ActiveRecord::Import::Result.new([], 0, [], [])
-          # supports 2-element array and array
-        elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
-
-          unless args.last.first.is_a?(Array)
-            raise ArgumentError, "Last argument should be a two dimensional array '[[]]'. First element in array was a #{args.last.first.class}"
-          end
-
-          column_names, array_of_attributes = args
-
-          # dup the passed args so we don't modify unintentionally
-          column_names = column_names.dup
-          array_of_attributes = array_of_attributes.map(&:dup)
+          models, column_names, default_column_names = models_and_column_names_for_import(conn, args)
+          array_of_attributes = attributes_from_models(conn, models, column_names, validator, options)
         else
-          raise ArgumentError, "Invalid arguments!"
+          column_names, array_of_attributes = column_names_and_attributes_for_import(args)
         end
 
-        # Force the primary key col into the insert if it's not
-        # on the list and we are using a sequence and stuff a nil
-        # value for it into each row so the sequencer will fire later
-        symbolized_column_names = Array(column_names).map(&:to_sym)
-        symbolized_primary_key = Array(primary_key).map(&:to_sym)
-
-        if !symbolized_primary_key.to_set.subset?(symbolized_column_names.to_set) && conn.prefetch_primary_key? && sequence_name
-          column_count = column_names.size
-          column_names.concat(Array(primary_key)).uniq!
-          columns_added = column_names.size - column_count
-          new_fields = Array.new(columns_added)
-          array_of_attributes.each { |a| a.concat(new_fields) }
-        end
+        # Rows without a value for these columns get the database's default rather than NULL
+        options[:database_default_columns] = Array(default_column_names) | database_populated_primary_keys(conn)
 
         # Don't modify incoming arguments
         on_duplicate_key_update = options[:on_duplicate_key_update]
         if on_duplicate_key_update
-          updatable_columns = symbolized_column_names.reject { |c| symbolized_primary_key.include? c }
-          options[:on_duplicate_key_update] = if on_duplicate_key_update.is_a?(Hash)
-            on_duplicate_key_update.each_with_object({}) do |(k, v), duped_options|
-              duped_options[k] = if k == :columns && v == :all
-                updatable_columns
-              elsif v.duplicable?
-                v.dup
-              else
-                v
-              end
-            end
-          elsif on_duplicate_key_update == :all
-            updatable_columns
-          elsif on_duplicate_key_update.duplicable?
-            on_duplicate_key_update.dup
-          else
-            on_duplicate_key_update
-          end
+          options[:on_duplicate_key_update] = on_duplicate_key_update_for_import(on_duplicate_key_update, column_names, options)
         end
 
+        add_primary_key_for_sequence(conn, column_names, array_of_attributes)
+
         timestamps = {}
 
         # record timestamps unless disabled in ActiveRecord::Base
@@ -737,41 +604,20 @@ class ActiveRecord::Base
         return_obj = if is_validating
           import_with_validations( conn, column_names, array_of_attributes, options ) do |failed_instances|
             if models
-              models.each_with_index do |m, i|
-                next unless m.errors.any?
-
-                failed_instances << (options[:track_validation_failures] ? [i, m] : m)
-              end
+              add_failed_models(failed_instances, models, options)
             else
-              # create instances for each of our column/value sets
-              arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
-
-              # keep track of the instance and the position it is currently at. if this fails
-              # validation we'll use the index to remove it from the array_of_attributes
-              arr.each_with_index do |hsh, i|
-                # utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
-                model = new do |m|
-                  hsh.each_pair { |k, v| m[k] = v }
-                end
-
-                next if validator.valid_model?(model)
-                raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
-
-                array_of_attributes[i] = nil
-                failure = model.dup
-                failure.errors.send(:initialize_dup, model.errors)
-                failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
-              end
-              array_of_attributes.compact!
+              validate_attributes_for_import(failed_instances, column_names, array_of_attributes, validator, options)
             end
           end
         else
           import_without_validations_or_callbacks( conn, column_names, array_of_attributes, options )
         end
 
+        return return_obj if options[:all_or_none] && return_obj.failed_instances.any?
+
         if options[:synchronize]
           sync_keys = options[:synchronize_keys] || Array(primary_key)
-          synchronize( options[:synchronize], sync_keys)
+          synchronized_instances = synchronize_imported_instances(options[:synchronize], sync_keys)
         end
         return_obj.num_inserts = 0 if return_obj.num_inserts.nil?
 
@@ -781,11 +627,14 @@ class ActiveRecord::Base
 
           # if there are auto-save associations on the models we imported that are new, import them as well
           if options[:recursive]
-            options[:on_duplicate_key_update] = on_duplicate_key_update unless on_duplicate_key_update.nil?
-            import_associations(models, options.dup.merge(validate: false))
+            import_associations_of_imported_models(models, return_obj, on_duplicate_key_update, options)
           end
         end
 
+        # apply the changes of synchronized instances that weren't marked clean above,
+        # such as when importing columns and values rather than models
+        synchronized_instances&.each { |instance| instance.changes_applied if instance.changed? }
+
         return_obj
       end
     end
@@ -820,8 +669,7 @@ class ActiveRecord::Base
       return ActiveRecord::Import::Result.new([], 0, [], []) if array_of_attributes.empty?
 
       column_names = column_names.map do |name|
-        original_name = attribute_alias?(name) ? attribute_alias(name) : name
-        original_name.to_sym
+        column_name_for_import(name).to_sym
       end
       scope_columns, scope_values = scope_attributes.to_a.transpose
 
@@ -849,7 +697,8 @@ class ActiveRecord::Base
       pre_sql_statements = conn.pre_sql_statements( options )
       insert_sql = ['INSERT', pre_sql_statements, "INTO #{quoted_table_name} #{columns_sql} VALUES "]
       insert_sql = insert_sql.flatten.join(' ')
-      values_sql = values_sql_for_columns_and_attributes(conn, columns, array_of_attributes)
+      values_sql = values_sql_for_columns_and_attributes(conn, columns, array_of_attributes, Array(options[:database_default_columns]))
+      comment_sql = sql_comment_for_annotations(conn)
 
       number_inserted = 0
       ids = []
@@ -859,6 +708,7 @@ class ActiveRecord::Base
       if supports_import?(conn)
         # generate the sql
         post_sql_statements = conn.post_sql_statements( quoted_table_name, options )
+        post_sql_statements += [comment_sql] if comment_sql
         import_size = values_sql.size
 
         batch_size = options[:batch_size] || import_size
@@ -877,17 +727,21 @@ class ActiveRecord::Base
             "#{model_name} Create Many" )
 
           number_inserted += result.num_inserts
-          ids += result.ids
-          results += result.results
+          ids.concat(result.ids)
+          results.concat(result.results)
           affected_rows = result.affected_rows.nil? ? nil : (affected_rows || 0) + result.affected_rows
           current_batch += 1
 
           progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
         end
       else
+        logger&.warn "activerecord-import: the #{conn.adapter_name} adapter doesn't support bulk import, so #{model_name} " \
+          "records are being inserted one at a time. If the adapter is supported under another name, require its " \
+          "activerecord-import adapter, e.g. activerecord-import/active_record/adapters/postgresql_adapter."
+
         transaction(requires_new: true) do
           values_sql.each do |values|
-            ids << conn.insert(insert_sql + values)
+            ids << conn.insert("#{insert_sql}#{values}#{comment_sql}", "#{model_name} Create Many")
             number_inserted += 1
           end
         end
@@ -897,6 +751,239 @@ class ActiveRecord::Base
 
     private
 
+    # Takes the options Hash, if any, off the end of +args+ and returns the options
+    # for the import, with their defaults, along with the arguments to import.
+    def options_and_arguments_for_import(args)
+      options = { validate: true, timestamps: true, track_validation_failures: false }
+      options.merge!( args.pop ) if args.last.is_a? Hash
+      args = arrays_from_array_like_import_arguments(args)
+      # making sure that the current model and its primary key are used, rather than
+      # the parent model's when importing associations recursively
+      options[:model] = self
+      options[:primary_key] = primary_key
+      options[:locking_column] = locking_column if locking_enabled?
+      [options, args]
+    end
+
+    # Returns the models to import from +args+, the column names to import them with,
+    # and the columns the database populates for the models that don't assign them,
+    # which is nil when +args+ give the column names.
+    def models_and_column_names_for_import(conn, args)
+      if args.length == 2
+        models = args.last
+        column_names = args.first.dup
+      else
+        models = args.first
+        column_names = if conn.try(:supports_virtual_columns?)
+          columns.reject(&:virtual?).map(&:name)
+        else
+          self.column_names.dup
+        end
+        populated_column_names = database_populated_column_names(conn, column_names, models)
+        # Leave out the ones no model assigns, and let the database fill in the rest for the models that don't
+        default_column_names = populated_column_names.select { |name| models.any? { |model| model.attribute_changed?(name) } }
+        column_names -= populated_column_names - default_column_names
+      end
+
+      # Leave out a primary key the database populates, like a uuid, only when no model has one, so
+      # assigned ids are kept. Otherwise the database fills it in for the models without one.
+      column_names -= database_populated_primary_keys(conn) if models.all? { |model| model.id.nil? }
+
+      [models, column_names, default_column_names]
+    end
+
+    # Returns the values of +column_names+ for each of +models+, the rows to import. Models
+    # that +validator+ finds invalid are left out, or raise ActiveRecord::RecordInvalid with
+    # the :raise_error option. Without a +validator+, the models aren't validated.
+    def attributes_from_models(conn, models, column_names, validator, options)
+      update_attrs = if record_timestamps && options[:timestamps]
+        send(:timestamp_attributes_for_update).map(&:to_sym)
+      end
+
+      array_of_attributes = []
+
+      models.each do |model|
+        if supports_setting_primary_key_of_imported_objects?(conn)
+          load_association_ids(model)
+        end
+
+        if validator && !validator.valid_model?(model)
+          raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
+          next
+        end
+
+        array_of_attributes << column_names.map do |name|
+          if model.persisted? &&
+             update_attrs && update_attrs.include?(name.to_sym) &&
+             !model.send("#{name}_changed?")
+            nil
+          else
+            name = name.to_s
+            # Read a column's own value, as saving the model would, even if it's also an alias
+            value = columns_hash.key?(name) ? model._read_attribute(name) : model.read_attribute(name)
+            model.instance_of?(self) ? value : value_from_model_type(model, name, value)
+          end
+        end
+      end
+
+      array_of_attributes
+    end
+
+    # Returns the column names and the rows to import from +args+, which are either an
+    # array of hashes, optionally after column names, or column names and an array of values.
+    def column_names_and_attributes_for_import(args)
+      # supports array of hash objects
+      if args.last.is_a?( Array ) && args.last.first.is_a?(Hash)
+        if args.length == 2
+          array_of_hashes = args.last
+          column_names = args.first.dup
+          allow_extra_hash_keys = true
+        else
+          array_of_hashes = args.first
+          column_names = array_of_hashes.first.keys
+          allow_extra_hash_keys = false
+        end
+
+        array_of_attributes = array_of_hashes.map do |h|
+          error_message = validate_hash_import(h, column_names, allow_extra_hash_keys)
+
+          raise ArgumentError, error_message if error_message
+
+          column_names.map do |key|
+            h[key]
+          end
+        end
+        # supports 2-element array and array
+      elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
+
+        unless args.last.first.is_a?(Array)
+          raise ArgumentError, "Last argument should be a two dimensional array '[[]]'. First element in array was a #{args.last.first.class}"
+        end
+
+        column_names, array_of_attributes = args
+
+        # dup the passed args so we don't modify unintentionally
+        column_names = column_names.dup
+        array_of_attributes = array_of_attributes.map(&:dup)
+      else
+        raise ArgumentError, invalid_import_arguments_message(args)
+      end
+
+      [column_names, array_of_attributes]
+    end
+
+    # Returns a copy of the +on_duplicate_key_update+ option to import with, so it can be
+    # changed, such as when adding timestamp columns, without modifying the option that was
+    # passed in. :all is replaced by the columns other than the primary key and the
+    # timestamps recorded on create.
+    def on_duplicate_key_update_for_import(on_duplicate_key_update, column_names, options)
+      symbolized_column_names = Array(column_names).map(&:to_sym)
+      symbolized_primary_key = Array(primary_key).map(&:to_sym)
+
+      updatable_columns = symbolized_column_names.reject { |c| symbolized_primary_key.include? c }
+      # Like the primary key, timestamps recorded on create shouldn't change when a record is updated
+      if record_timestamps && options[:timestamps]
+        updatable_columns -= timestamp_attributes_for_create_in_model.map(&:to_sym)
+      end
+
+      if on_duplicate_key_update.is_a?(Hash)
+        on_duplicate_key_update.each_with_object({}) do |(k, v), duped_options|
+          duped_options[k] = if k == :columns && v == :all
+            updatable_columns
+          elsif v.duplicable?
+            v.dup
+          else
+            v
+          end
+        end
+      elsif on_duplicate_key_update == :all
+        updatable_columns
+      elsif on_duplicate_key_update.duplicable?
+        on_duplicate_key_update.dup
+      else
+        on_duplicate_key_update
+      end
+    end
+
+    # Force the primary key col into the insert if it's not
+    # on the list and we are using a sequence and stuff a nil
+    # value for it into each row so the sequencer will fire later
+    def add_primary_key_for_sequence(conn, column_names, array_of_attributes)
+      symbolized_column_names = Array(column_names).map(&:to_sym)
+      symbolized_primary_key = Array(primary_key).map(&:to_sym)
+
+      if !symbolized_primary_key.to_set.subset?(symbolized_column_names.to_set) && conn.prefetch_primary_key? && sequence_name
+        column_count = column_names.size
+        column_names.concat(Array(primary_key)).uniq!
+        columns_added = column_names.size - column_count
+        new_fields = Array.new(columns_added)
+        array_of_attributes.each { |a| a.concat(new_fields) }
+      end
+    end
+
+    # Adds the models that failed validation, when their rows were read, to +failed_instances+.
+    def add_failed_models(failed_instances, models, options)
+      models.each_with_index do |m, i|
+        next unless m.errors.any?
+
+        failed_instances << (options[:track_validation_failures] ? [i, m] : m)
+      end
+    end
+
+    # Validates each row of +array_of_attributes+ as a new model, adding the models that
+    # fail to +failed_instances+ and removing their rows from +array_of_attributes+.
+    def validate_attributes_for_import(failed_instances, column_names, array_of_attributes, validator, options)
+      # create instances for each of our column/value sets
+      arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
+
+      # keep track of the instance and the position it is currently at. if this fails
+      # validation we'll use the index to remove it from the array_of_attributes
+      arr.each_with_index do |hsh, i|
+        # utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
+        model = new do |m|
+          hsh.each_pair do |k, v|
+            if write_id_column_directly?(k)
+              m._write_attribute("id", v)
+            else
+              m[k] = v
+            end
+          end
+        end
+
+        next if validator.valid_model?(model)
+        raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
+
+        array_of_attributes[i] = nil
+        failure = model.dup
+        failure.errors.send(:initialize_dup, model.errors)
+        failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
+      end
+      array_of_attributes.compact!
+    end
+
+    # Imports the new and changed associations of the +models+ that were imported, with the
+    # :on_duplicate_key_update option that was passed in rather than the copy used for +models+.
+    def import_associations_of_imported_models(models, import_result, on_duplicate_key_update, options)
+      options[:on_duplicate_key_update] = on_duplicate_key_update unless on_duplicate_key_update.nil?
+      import_associations(imported_models(models, import_result, options), options.dup.merge(validate: false))
+    end
+
+    # ActiveRecord treats "id" as the primary key when writing attributes
+    # through id= or []=. When the primary key is composite or is another
+    # column, an "id" column has to be written directly instead.
+    def write_id_column_directly?(column)
+      column.to_s == "id" && primary_key != "id"
+    end
+
+    # Returns SQL comments for the annotations on the current scope, as
+    # ActiveRecord adds them to its queries, or nil if there are none.
+    def sql_comment_for_annotations(conn)
+      annotations = all.annotate_values
+      return if annotations.empty?
+
+      " #{annotations.map { |value| "/* #{conn.sanitize_as_sql_comment(value)} */" }.join(' ')}"
+    end
+
     def associated_options(options, association)
       return options unless options.key?(:recursive_on_duplicate_key_update)
 
@@ -905,9 +992,55 @@ class ActiveRecord::Base
       )
     end
 
+    # Returns the models that weren't rejected by validations.
+    def imported_models(models, import_result, options)
+      failed_instances = import_result.failed_instances
+      failed_instances = failed_instances.map(&:last) if options[:track_validation_failures]
+      models - failed_instances
+    end
+
+    # Synchronizes +instances+ with the database like ActiveRecord::Base.synchronize,
+    # but rather than discarding their changes, it assigns the database values
+    # on top of them. Once the instances are marked clean, previous_changes then
+    # reflects what the import changed, as it does without synchronizing.
+    # Instances left without changes are marked clean right away. Returns the
+    # instances that were synchronized.
+    def synchronize_imported_instances(instances, keys)
+      synchronize_instances(instances, keys) do |instance, fresh_attributes|
+        attributes = instance.instance_variable_get(:@attributes)
+        # the attributes of destroyed records are frozen
+        attributes = attributes.deep_dup if attributes.frozen?
+
+        names = fresh_attributes.keys
+        if names.all? { |name| attributes.key?(name) }
+          names.each do |name|
+            value = fresh_attributes.fetch_value(name)
+            attributes.write_from_user(name, value) unless same_database_value?(attributes.fetch_value(name), value)
+          end
+          instance.instance_variable_set :@attributes, attributes
+          instance.changes_applied unless instance.changed?
+        else
+          # the instance doesn't have all of the attributes, such as when it was
+          # loaded with select, so take the record's attributes as they are
+          instance.instance_variable_set :@attributes, fresh_attributes
+          instance.clear_changes_information
+        end
+      end
+    end
+
+    # Returns true if +database_value+, reloaded from the database, is the +value+ an instance
+    # already has. Databases store times to the microsecond, so a time with more precision,
+    # which ActiveRecord < 7.0 keeps in memory, comes back rounded or truncated.
+    def same_database_value?(value, database_value)
+      return true if value == database_value
+      return false unless value.is_a?(Time) && database_value.is_a?(Time)
+
+      value.round(6) == database_value || value.floor(6) == database_value
+    end
+
     def set_attributes_and_mark_clean(conn, models, import_result, timestamps, options)
       return if models.nil?
-      models -= import_result.failed_instances
+      models = imported_models(models, import_result, options)
 
       # if ids were returned for all models we know all were updated
       if models.size == import_result.ids.size
@@ -924,20 +1057,18 @@ class ActiveRecord::Base
       deserialize_value = lambda do |column, value|
         column = schema_columns_hash(conn)[column]
         return value unless column
-        if respond_to?(:type_caster)
-          type = type_for_attribute(column.name)
-          type.deserialize(value)
-        elsif column.respond_to?(:type_cast_from_database)
-          column.type_cast_from_database(value)
-        else
-          value
-        end
+        type = type_for_attribute(column.name)
+        type.deserialize(value)
       end
 
       set_value = lambda do |model, column, value|
         val = deserialize_value.call(column, value)
         if model.attribute_names.include?(column)
-          model.send("#{column}=", val)
+          if write_id_column_directly?(column)
+            model._write_attribute(column, val)
+          else
+            model.send("#{column}=", val)
+          end
         else
           attributes = attributes_builder.build_from_database(model.attributes.merge(column => val))
           model.instance_variable_set(:@attributes, attributes)
@@ -962,14 +1093,7 @@ class ActiveRecord::Base
       end
 
       models.each do |model|
-        if model.respond_to?(:changes_applied) # Rails 4.1.8 and higher
-          model.changes_internally_applied if model.respond_to?(:changes_internally_applied) # legacy behavior for Rails 5.1
-          model.changes_applied
-        elsif model.respond_to?(:clear_changes_information) # Rails 4.0 and higher
-          model.clear_changes_information
-        else # Rails 3.2
-          model.instance_variable_get(:@changed_attributes).clear
-        end
+        model.changes_applied
         model.instance_variable_set(:@new_record, false)
       end
     end
@@ -989,12 +1113,24 @@ class ActiveRecord::Base
           association = association.target
           next if association.blank? || model.public_send(column_name).present?
 
-          association_primary_key = Array(association_reflection.association_primary_key.tr("[]:", "").split(", "))[column_index]
+          association_primary_key = association_primary_key_names(association_reflection)[column_index]
+          next unless association_primary_key
+
           model.public_send("#{column_name}=", association.send(association_primary_key))
         end
       end
     end
 
+    # Returns the names of the columns that a belongs_to association's foreign key refers to.
+    # For a composite key, ActiveRecord 7.1+ returns an Array, while older versions return
+    # its string form, e.g. "[:account_id, :id]".
+    def association_primary_key_names(association_reflection)
+      association_primary_key = association_reflection.association_primary_key
+      return association_primary_key.map(&:to_s) if association_primary_key.is_a?(Array)
+
+      association_primary_key.tr("[]:", "").split(", ")
+    end
+
     def import_associations(models, options)
       # now, for all the dirty associations, collect them into a new set of models, then recurse.
       # notes:
@@ -1020,8 +1156,45 @@ class ActiveRecord::Base
       end
     end
 
+    # Returns the columns the database populates itself, such as with a default
+    # function, when importing new +models+. Like ActiveRecord's own inserts, the
+    # database populates these for models that don't assign a value, instead of
+    # inserting NULL. The primary key is handled separately.
+    def database_populated_column_names(conn, column_names, models)
+      return [] if models.any?(&:persisted?)
+
+      columns_hash = schema_columns_hash(conn)
+      primary_keys = Array(primary_key).map(&:to_s)
+
+      column_names.select do |name|
+        column = columns_hash[name.to_s]
+        next false if column.nil? || primary_keys.include?(name.to_s)
+
+        column.respond_to?(:auto_populated?) ? column.auto_populated? : column.default_function
+      end
+    end
+
+    # Returns the primary key columns the database populates itself, such as a uuid
+    # or a key with a default function, rather than from the model's sequence.
+    def database_populated_primary_keys(conn)
+      Array(primary_key).select do |name|
+        column = schema_columns_hash(conn)[name]
+        column && (column.type == :uuid || (column.default_function && sequence_name.blank?))
+      end
+    end
+
+    # A model of another class, such as an STI subclass with its own enum, can give
+    # a column a different type than this class, which serializes the imported values.
+    # Convert the model's value to this class's type through its database value.
+    def value_from_model_type(model, name, value)
+      name = column_name_for_import(name)
+      model_type = model.class.type_for_attribute(name)
+      type = type_for_attribute(name)
+      model_type.equal?(type) ? value : type.deserialize(model_type.serialize(value))
+    end
+
     def schema_columns_hash(conn)
-      if respond_to?(:ignored_columns) && ignored_columns.any?
+      if ignored_columns.any?
         conn.schema_cache.columns_hash(table_name)
       else
         columns_hash
@@ -1046,18 +1219,25 @@ class ActiveRecord::Base
         # Wrap target in an array if not already
         association = Array(association.target)
 
-        changed_objects = association.select { |a| a.new_record? || a.changed? }
+        foreign_key_columns = Array(association_reflection.inverse_of&.foreign_key || association_reflection.foreign_key)
+        foreign_key_values = association_foreign_key_values(model, association_reflection)
+
+        # For polymorphic associations
+        association_name = model.class.polymorphic_name
+
+        # Import new and changed records, and existing records that were assigned to
+        # this model but aren't linked to it yet, as saving the model would
+        changed_objects = association.select do |child|
+          child.new_record? || child.changed? ||
+            (!association_reflection.through_reflection? &&
+              reassigned_to_parent?(child, foreign_key_columns, foreign_key_values, association_reflection.type, association_name))
+        end
+
         changed_objects.each do |child|
-          Array(association_reflection.inverse_of&.foreign_key || association_reflection.foreign_key).each_with_index do |column, index|
-            child.public_send("#{column}=", Array(model.id)[index])
+          foreign_key_columns.each_with_index do |column, index|
+            child.public_send("#{column}=", foreign_key_values[index])
           end
 
-          # For polymorphic associations
-          association_name = if model.class.respond_to?(:polymorphic_name)
-            model.class.polymorphic_name
-          else
-            model.class.base_class
-          end
           association_reflection.type.try do |type|
             child.public_send("#{type}=", association_name)
           end
@@ -1067,33 +1247,55 @@ class ActiveRecord::Base
       associated_objects_by_class
     end
 
+    # Returns the values of +model+ that the foreign key of +association_reflection+
+    # refers to: the columns named by the association's :primary_key option when
+    # one is given (e.g. <tt>has_many :reviews, primary_key: :identifier</tt>),
+    # and the model's id otherwise.
+    def association_foreign_key_values(model, association_reflection)
+      if association_reflection.options[:primary_key]
+        Array(association_reflection.active_record_primary_key).map { |key| model.read_attribute(key) }
+      else
+        Array(model.id)
+      end
+    end
+
+    # Returns true if +child+, an existing record of a has_one or has_many association,
+    # doesn't refer to the parent yet, such as when it was assigned to a new parent.
+    def reassigned_to_parent?(child, foreign_key_columns, foreign_key_values, type_column, type_name)
+      return false if child.new_record?
+
+      parent_values = foreign_key_columns.zip(foreign_key_values)
+      parent_values << [type_column, type_name.to_s] if type_column.present?
+      parent_values.any? { |column, value| child.has_attribute?(column) && child.read_attribute(column) != value }
+    end
+
     # Returns SQL the VALUES for an INSERT statement given the passed in +columns+
-    # and +array_of_attributes+.
-    def values_sql_for_columns_and_attributes(conn, columns, array_of_attributes) # :nodoc:
+    # and +array_of_attributes+. Nil values for +default_column_names+ use the
+    # database's default.
+    def values_sql_for_columns_and_attributes(conn, columns, array_of_attributes, default_column_names = []) # :nodoc:
       array_of_attributes.map do |arr|
+        if arr.length < columns.length
+          raise ArgumentError, "Number of values (#{arr.length}) is less than number of columns (#{columns.length})"
+        end
+
         my_values = arr.each_with_index.map do |val, j|
           column = columns[j]
 
           # be sure to query sequence_name *last*, only if cheaper tests fail, because it's costly
           if val.nil? && Array(primary_key).first == column.name && !sequence_name.blank?
             conn.next_value_for_sequence(sequence_name)
+          elsif val.nil? && default_column_names.include?(column.name)
+            unless conn.supports_default_keyword_in_values?
+              raise ArgumentError, "#{conn.adapter_name} doesn't support DEFAULT in VALUES, so the database can't populate " \
+                "#{table_name}.#{column.name} for only the rows without a value. Import those rows separately so the column can be left out."
+            end
+            "DEFAULT"
           elsif val.respond_to?(:to_sql)
             "(#{val.to_sql})"
           elsif column
-            if respond_to?(:type_caster)                                         # Rails 5.0 and higher
-              type = type_for_attribute(column.name)
-              val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
-              conn.quote(val)
-            elsif column.respond_to?(:type_cast_from_user)                       # Rails 4.2
-              conn.quote(column.type_cast_from_user(val), column)
-            else                                                                 # Rails 3.2, 4.0 and 4.1
-              if serialized_attributes.include?(column.name)
-                val = serialized_attributes[column.name].dump(val)
-              end
-              # Fixes #443 to support binary (i.e. bytea) columns on PG
-              val = column.type_cast(val) unless column.type && column.type.to_sym == :binary
-              conn.quote(val, column)
-            end
+            type = type_for_attribute(column.name)
+            val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
+            conn.quote(val)
           else
             raise ArgumentError, "Number of values (#{arr.length}) exceeds number of columns (#{columns.length})"
           end
@@ -1106,14 +1308,8 @@ class ActiveRecord::Base
       timestamp_columns = {}
       timestamps        = {}
 
-      if respond_to?(:all_timestamp_attributes_in_model, true) # Rails 5.1 and higher
-        timestamp_columns[:create] = timestamp_attributes_for_create_in_model
-        timestamp_columns[:update] = timestamp_attributes_for_update_in_model
-      else
-        instance = allocate
-        timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
-        timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
-      end
+      timestamp_columns[:create] = timestamp_attributes_for_create_in_model
+      timestamp_columns[:update] = timestamp_attributes_for_update_in_model
 
       # use tz as set in ActiveRecord::Base
       default_timezone = if ActiveRecord.respond_to?(:default_timezone)
@@ -1146,6 +1342,17 @@ class ActiveRecord::Base
       timestamps
     end
 
+    # Converts array-like arguments, such as a Relation or a Set, to arrays.
+    def arrays_from_array_like_import_arguments(args) # :nodoc:
+      args.map { |arg| arg.is_a?(Enumerable) && !arg.is_a?(Array) && !arg.is_a?(Hash) ? arg.to_a : arg }
+    end
+
+    def invalid_import_arguments_message(args) # :nodoc:
+      received = args.map { |arg| arg.is_a?(Array) && arg.any? ? "Array of #{arg.first.class}" : arg.class.name }
+      "Invalid arguments! Expected an array of models, an array of hashes, or column names and an array " \
+        "of values, but received #{received.empty? ? 'no arguments' : received.join(', ')}"
+    end
+
     # Returns an Array of Hashes for the passed in +column_names+ and +array_of_attributes+.
     def validations_array_for_column_names_and_attributes( column_names, array_of_attributes ) # :nodoc:
       array_of_attributes.map { |values| Hash[column_names.zip(values)] }