activerecord-import 2.3.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 18ab977c921d6e9c51ca329d136998d858ddbe3cae0a186160886cd43ee6b6e9
-  data.tar.gz: 98d166b037d997de14492f621099ffb63af16505da1c8c36a2e302a903213777
+  metadata.gz: 44eca8fc38325819cfc2eeb8c9a6151182f5de96135af6ad3ae1cc145c5f7740
+  data.tar.gz: 9f6ef423efde4813da41b8eb5ba218d4166bb3a86beba83d72b4bbf9e70af27f
 SHA512:
-  metadata.gz: 2a593859fb0b568b88e29e99e849257b536a78b87c9557e1f98fe196a8c1c0be3116c88676c74c58044f76cb69a2ae17a6093fcc463bd2ad5d914f551229c9b2
-  data.tar.gz: 10b347057b09a027bbaafc752363553ed117ada314ced7c383276863b9989eda8ec92e9eabee8697099c5b6db7b6e909002a7f86d40df821438b9dabadb15f55
+  metadata.gz: f534b0b2a47c29484b8957109794ab8924c758993e93ecd871104f97ce8402856999c40d1e845ff7a2857fb92e12291a0d93259a9bdd17d4b60f81cb20ac5e42
+  data.tar.gz: f9e8de71d80edd76c978a1d85807b6a94247da8366dfda54a4c1e2c01ed25c31767d836c2ec487d39377379492dd896f320a986c1a158cfda7f810e7b3ba3a80

data/CHANGELOG.md

@@ -1,3 +1,94 @@
+## Changes in 3.0.0
+
+### Breaking Changes
+
+* Drop support for Ruby < 3.1 and ActiveRecord < 6.1. via #918 and #943.
+* MariaDB 10.5+ now imports with `INSERT ... RETURNING`, as PostgreSQL does. Imports return
+  `ids` and `results`, imported models get their ids and are marked persisted, and
+  `recursive: true` imports associations (it was ignored on MariaDB before). MariaDB doesn't
+  report affected rows for `INSERT ... RETURNING`, so `affected_rows` is now `nil` on MariaDB
+  10.5+ unless `no_returning: true` is passed. As on PostgreSQL, `on_duplicate_key_ignore` is
+  ignored for recursive imports. MySQL and MariaDB before 10.5 are unchanged. via #926.
+* When only some rows have a value for a column the database fills in (a uuid or other primary
+  key with a default function, or a column with a default function), the other rows now get the
+  database's default instead of `NULL` on PostgreSQL and MySQL/MariaDB, and assigned primary keys
+  are no longer dropped when the first model doesn't have one. SQLite can't use `DEFAULT` in
+  `VALUES`, so such a mixed import now raises `ArgumentError` there. Fixes #702, #777, #805 and
+  #827 via #929.
+* `on_duplicate_key_update: :all` and `columns: :all` no longer update the timestamps recorded on
+  create (`created_at`, `created_on`) when import records timestamps. via #927.
+* Recursive imports now also import existing records that were assigned to a new parent, so they're
+  linked to it as `save` would do. Without `on_duplicate_key_update` that raises a duplicate key
+  error, where the records used to be silently left unlinked. Fixes #804 via #939.
+* Importing with `synchronize` now keeps the synchronized models' `previous_changes`, rather than
+  clearing them. Fixes #738 via #933.
+
+### New Features
+
+* Add `require "activerecord-import/without_aliases"` to load the library without the `import` and
+  `import!` aliases, for gems such as `elasticsearch-model` that define their own `import`. Use
+  `bulk_import`/`bulk_import!` instead. Fixes #592 via #915.
+* `import` accepts array-like arguments such as an `ActiveRecord::Relation` or a `Set`, and invalid
+  arguments now raise an error saying what was received. Fixes #785 via #935.
+* Annotations from the current scope (`Model.annotate(...)`) are added to the INSERT statements as
+  SQL comments. Fixes #872 via #937.
+* When activerecord-import is required after the database connection is established, import now
+  loads the adapter's import support itself instead of inserting one row at a time. Adapters
+  without bulk import support log a warning when import falls back to row-by-row inserts.
+  via #925 and #941.
+* Validate imported models with `valid?`, so `around` validate callbacks are supported. via #919.
+
+### Fixes
+
+* Report `affected_rows` with the mysql2 adapter. It was always `nil` with mysql2, and only
+  reported with trilogy. via #944.
+* Clear the query cache after MySQL imports on ActiveRecord < 7.1. via #926.
+* Don't import the associations of models that failed validation. Fixes #857. Thanks to
+  @jacob-carlborg-apoex via #858.
+* Fix `NoMethodError` in `returning_selections` when there's no model. Thanks to @mirelon via #892.
+* Don't mark models that failed validation as persisted when `track_validation_failures` is set.
+  Fixes #888. Thanks to @OskarEichler via #903.
+* Leave models unchanged when `all_or_none` aborts an import. Thanks to @OskarEichler via #904.
+* Include the owner's foreign key and polymorphic type when importing models with explicit columns
+  through an association. Thanks to @OskarEichler via #905.
+* Support string hash keys in association imports. Thanks to @OskarEichler via #909.
+* Use the canonical polymorphic name (e.g. for STI owners) in association imports. Thanks to
+  @OskarEichler via #910.
+* Raise `ArgumentError` for rows with fewer values than columns, before running any SQL. Thanks
+  to @OskarEichler via #912.
+* Quote PostgreSQL `RETURNING` column names and generated conflict targets. Thanks to
+  @OskarEichler via #907 and #908.
+* Skip uniqueness validations on associated records unless `validate_uniqueness: true`. Fixes #841
+  via #916.
+* Use each model's own class for `options[:model]` in recursive imports. via #920.
+* Leave columns the database populates (e.g. a uuid with a default function) out of the INSERT
+  when no imported model assigns them, as `save` does. via #921.
+* Don't follow an attribute alias when its name is also a column. via #922.
+* Don't modify the options hash passed to `import!`. via #924.
+* Don't send a `RETURNING` clause on PostgreSQL with `no_returning: true`. Fixes #826 via #930.
+* Support `returning: :id` with composite primary keys that include an `id` column. Fixes #842
+  via #931.
+* Use the association's `primary_key` for foreign keys in recursive imports. Fixes #878 via #932.
+* Serialize values of STI subclass models with the subclass's attribute types, e.g. for enums
+  defined only on the subclass. Fixes #789 via #934.
+* Fix `NoMethodError` when filling in composite `belongs_to` foreign keys. via #938.
+
+### Performance
+
+* Match synchronized records in linear rather than quadratic time. Thanks to @OskarEichler via #902.
+* Collect returned ids and results across batches without recopying them. Thanks to @OskarEichler
+  via #906.
+
+### Development
+
+* Test against MariaDB 10.11 and 11.8 in CI. via #926.
+* Run the composite primary key tests on ActiveRecord 7.1, and speed up the JRuby CI job.
+  via #923, #928 and #945.
+* Run the database tests locally against Docker without a hand-written `test/database.yml`.
+  via #945.
+* Document the memory use of large imports and what gets logged. via #940.
+* Split `import_helper` into smaller steps. via #947.
+
 ## Changes in 2.3.0
 
 ### New Features

data/README.markdown

@@ -37,6 +37,7 @@ The gem provides the following high-level features:
   * [ActiveRecord Models](#activerecord-models)
   * [Batching](#batching)
   * [Recursive](#recursive)
+  * [Annotating the SQL](#annotating-the-sql)
 * [Options](#options)
   * [Duplicate Key Ignore](#duplicate-key-ignore)
   * [Duplicate Key Update](#duplicate-key-update)
@@ -44,6 +45,7 @@ The gem provides the following high-level features:
 * [Counter Cache](#counter-cache)
 * [ActiveRecord Timestamps](#activerecord-timestamps)
 * [Callbacks](#callbacks)
+* [Logging](#logging)
 * [Supported Adapters](#supported-adapters)
 * [Additional Adapters](#additional-adapters)
 * [Requiring](#requiring)
@@ -231,6 +233,16 @@ columns = [ :title ]
 Book.import columns, books, batch_size: 2
 ```
 
+`batch_size` only controls how many rows go into each INSERT statement. The whole import is still prepared up front: every record or row passed to `import`, and the SQL for all of them, is held in memory until the import finishes. For very large imports, load and import the data in chunks, so only one chunk is in memory at a time:
+
+```ruby
+Source.find_in_batches(batch_size: 1_000) do |sources|
+  Book.import sources.map { |source| Book.new(title: source.title, author: source.author) }
+end
+```
+
+Each `import` call validates, inserts and returns its own result, so options like `all_or_none` apply to each chunk rather than to all of the data.
+
 If your import is particularly large or slow (possibly due to [callbacks](#callbacks)) whilst batch importing, you might want a way to report back on progress. This is supported by passing a callable as the `batch_progress` option. e.g:
 
 ```ruby
@@ -246,7 +258,7 @@ Book.import columns, books, batch_size: 2, batch_progress: my_proc
 #### Recursive
 
 > **Note**
-> This only works with PostgreSQL and ActiveRecord objects. This won't work with hashes or arrays as recursive inputs.
+> This only works with PostgreSQL and MariaDB 10.5+, and with ActiveRecord objects. This won't work with hashes or arrays as recursive inputs.
 
 Assume that Books <code>has_many</code> Reviews.
 
@@ -260,12 +272,25 @@ end
 Book.import books, recursive: true
 ```
 
+#### Annotating the SQL
+
+To add SQL comments to an import, import through a relation annotated with ActiveRecord's [`annotate`](https://api.rubyonrails.org/classes/ActiveRecord/QueryMethods.html#method-i-annotate), or within its `scoping` block. The comments are added to every INSERT statement, including when the import is batched or split into several statements.
+
+```ruby
+Book.annotate("nightly import").import books
+# INSERT INTO "books" (...) VALUES (...) /* nightly import */
+
+Book.annotate("nightly import").scoping do
+  Book.import books
+end
+```
+
 ### Options
 
 Key                       | Options               | Default            | Description
 ------------------------- | --------------------- | ------------------ | -----------
 :validate                 | `true`/`false`        | `true`             | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
-:validate_uniqueness      | `true`/`false`        | `false`            | Whether or not to run ActiveRecord uniqueness validations. Beware this will incur an sql query per-record (N+1 queries). (requires `>= v0.27.0`).
+:validate_uniqueness      | `true`/`false`        | `false`            | Whether or not to run ActiveRecord uniqueness validations, including on associated records that are validated with the model. Beware this will incur an sql query per-record (N+1 queries). (requires `>= v0.27.0`).
 :validate_with_context    | `Symbol`              |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
 :track_validation_failures| `true`/`false`        | `false`            | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]`
 :on_duplicate_key_ignore  | `true`/`false`        | `false`            | Allows skipping records with duplicate keys. See [here](#duplicate-key-ignore) for more details.
@@ -273,9 +298,9 @@ Key                       | Options               | Default            | Descrip
 :on_duplicate_key_update  | :all, `Array`, `Hash` | N/A                | Allows upsert logic to be used. See [here](#duplicate-key-update) for more details.
 :synchronize              | `Array`               | N/A                | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
 :timestamps               | `true`/`false`        | `true`             | Enables/disables timestamps on imported records.
-:recursive                | `true`/`false`        | `false`            | Imports has_many/has_one associations (PostgreSQL only).
+:recursive                | `true`/`false`        | `false`            | Imports has_many/has_one associations (PostgreSQL and MariaDB 10.5+ only).
 :recursive_on_duplicate_key_update | `Hash`       | N/A                | Allows upsert logic to be used for recursive associations. The hash key is the association name and the value has the same options as `:on_duplicate_key_update`. See [here](#duplicate-key-update) for more details.
-:batch_size               | `Integer`             | total # of records | Max number of records to insert per import
+:batch_size               | `Integer`             | total # of records | Max number of records to insert per INSERT statement. See [Batching](#batching).
 :raise_error              | `true`/`false`        | `false`            | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
 :all_or_none              | `true`/`false`        | `false`            | Will not import any records if there is a record with validation errors.
 
@@ -296,7 +321,7 @@ book.reload.title  # => "Book1"     (stayed the same)
 book.reload.author # => "George Orwell" (stayed the same)
 ```
 
-The option `:on_duplicate_key_ignore` is bypassed when `:recursive` is enabled for [PostgreSQL imports](https://rubydoc.info/gems/activerecord-import/ActiveRecord/Base#label-Options) (`:recursive` is not supported with other adapters).
+The option `:on_duplicate_key_ignore` is bypassed when `:recursive` is enabled for [PostgreSQL and MariaDB 10.5+ imports](https://rubydoc.info/gems/activerecord-import/ActiveRecord/Base#label-Options) (`:recursive` is not supported with other databases).
 
 #### Duplicate Key Update
 
@@ -416,7 +441,7 @@ Book.import books, validate_uniqueness: true
 
 ### Return Info
 
-The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed.
+The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres and MariaDB 10.5+, there will be two arrays `ids` and `results` that can be accessed. These are populated with `INSERT ... RETURNING`, which also sets the ids of imported models. On MariaDB this means `affected_rows` isn't available, since MariaDB doesn't report it for `INSERT ... RETURNING`; pass `no_returning: true` to get `affected_rows` instead of `ids`.
 
 ```ruby
 articles = [
@@ -434,11 +459,11 @@ demo.num_inserts
 => 1,
 
 demo.ids
-=> ["1", "2"] # for Postgres
+=> ["1", "2"] # for Postgres and MariaDB 10.5+
 => [] # for other DBs
 
 demo.results
-=> ["First Article", "Second Article"] # for Postgres
+=> ["First Article", "Second Article"] # for Postgres and MariaDB 10.5+
 => [] # for other DBs
 ```
 
@@ -499,12 +524,23 @@ end
 Book.import valid_books, validate: false
 ```
 
+### Logging
+
+Imports are logged like ActiveRecord's own queries, named `<Model> Create Many`. The imported values are part of the SQL rather than bind parameters, so a logged INSERT includes every value it imports. That can make for very long log lines, and ActiveRecord's `filter_attributes` doesn't apply to them, so the values of sensitive columns appear in the log too. ActiveRecord's own `insert_all` and `upsert_all` log the same way.
+
+If that's a concern, silence the logger around imports. For example, with Rails' logger:
+
+```ruby
+ActiveRecord::Base.logger.silence { Book.import books }
+```
+
 ### Supported Adapters
 
 The following database adapters are currently supported:
 
 * MySQL - supports core import functionality plus on duplicate key update support (included in activerecord-import 0.1.0 and higher)
 * MySQL2 - supports core import functionality plus on duplicate key update support (included in activerecord-import 0.2.0 and higher)
+* MariaDB - supported through the MySQL2 and Trilogy adapters, with the same functionality as MySQL. MariaDB 10.5+ also supports `ids`, `returning` and recursive imports.
 * PostgreSQL - supports core import functionality (included in activerecord-import 0.1.0 and higher)
 * SQLite3 - supports core import functionality (included in activerecord-import 0.1.0 and higher)
 * Oracle - supports core import functionality through DML trigger (available as an external gem: [activerecord-import-oracle_enhanced](https://github.com/keeguon/activerecord-import-oracle_enhanced))
@@ -550,12 +586,15 @@ You may want to manually load activerecord-import for one reason or another. Fir
 gem 'activerecord-import', require: false
 ```
 
-This will allow you to load up activerecord-import in the file or files where you are using it and only load the parts you need.
-If you are doing this within Rails and ActiveRecord has established a database connection (such as within a controller), you will need to do extra initialization work:
+This will allow you to load up activerecord-import in the file or files where you are using it and only load the parts you need:
 
 ```ruby
 require 'activerecord-import/base'
-# load the appropriate database adapter (postgresql, mysql2, sqlite3, etc)
+```
+
+This works even if ActiveRecord has already established a database connection (such as within a controller): the import support for the connection's adapter (postgresql, mysql2, sqlite3, etc) is loaded the first time you import. If your adapter goes by a name activerecord-import doesn't recognize, load its support yourself:
+
+```ruby
 require 'activerecord-import/active_record/adapters/postgresql_adapter'
 ```
 
@@ -599,6 +638,16 @@ When rubygems pushes the `lib` folder onto the load path a `require` will now fi
 
 Activerecord-Import adds the `.import` method onto `ActiveRecord::Base`. There are other gems, such as `elasticsearch-rails`, that do the same thing. In conflicts such as this, there is an aliased method named `.bulk_import` that can be used interchangeably.
 
+Some gems, such as `elasticsearch-model`, won't define their own `.import` on a model if one already exists. To load activerecord-import without the `import` and `import!` aliases, so only `bulk_import` and `bulk_import!` are defined, require it like so:
+
+```ruby
+gem 'activerecord-import', require: 'activerecord-import/without_aliases'
+```
+
+This takes precedence even if `activerecord-import` or `activerecord-import/base` is also required, for example by another gem, regardless of the order they're required in. To confirm the aliases aren't defined, check that `ActiveRecord::Base.respond_to?(:import)` returns `false` in a console.
+
+If you're loading activerecord-import manually (see [Manually Loading](#manually-loading)), require `activerecord-import/without_aliases` in place of `activerecord-import/base`.
+
 If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by explicitly setting this value within the model. For example:
 
 ```ruby
@@ -632,22 +681,36 @@ To document new information, please add to the README instead of the wiki. See h
 
 #### Running Tests
 
-The first thing you need to do is set up your database(s):
+The tests run against each database adapter (`bundle exec rake -T` lists them) and against multiple ActiveRecord versions, chosen with `AR_VERSION`. The SQLite tests need no setup. For MySQL, MariaDB and PostgreSQL, the quickest way to get databases is Docker (see [Docker Setup](#docker-setup)):
+
+```bash
+docker compose up -d
+rm -f Gemfile.lock
+AR_VERSION=8.1 bundle install
+AR_VERSION=8.1 bundle exec rake test:sqlite3 test:postgresql test:mysql2 test:trilogy
+```
 
-* copy `test/database.yml.sample` to `test/database.yml`
-* modify `test/database.yml` for your database settings
-* create databases as needed
+Without a `test/database.yml`, the tests use `test/database.default.yml`, which connects to the Docker databases by default and can be pointed elsewhere with environment variables:
 
-After that, you can run the tests. They run against multiple tests and ActiveRecord versions.
+| Variable | Default | |
+| --- | --- | --- |
+| `DB_HOST` | `127.0.0.1` | Host for all databases |
+| `DB_DATABASE` | `activerecord_import_test` | Database name |
+| `MYSQL_HOST`, `PG_HOST` | `DB_HOST` | Host for MySQL/MariaDB or PostgreSQL |
+| `MYSQL_PORT` | `13306` | MySQL/MariaDB port; Docker's MariaDB is on `13307` |
+| `PG_PORT` | `15432` | PostgreSQL port |
+| `MYSQL_USERNAME`, `MYSQL_PASSWORD` | `root`, empty | |
+| `PG_USERNAME`, `PG_PASSWORD` | `postgres`, empty | |
 
-This is one example of how to run the tests:
+For example, to run the MySQL tests against Docker's MariaDB, or against a MySQL server you run yourself:
 
 ```bash
-rm Gemfile.lock
-AR_VERSION=7.0 bundle install
-AR_VERSION=7.0 bundle exec rake test:postgresql test:sqlite3 test:mysql2
+MYSQL_PORT=13307 AR_VERSION=8.1 bundle exec rake test:mysql2
+MYSQL_PORT=3306 MYSQL_PASSWORD=secret AR_VERSION=8.1 bundle exec rake test:mysql2
 ```
 
+The PostgreSQL tests need the `hstore`, `pgcrypto` and `uuid-ossp` extensions, which they create themselves when connecting as a superuser, and `test:postgis` also needs PostGIS. If you'd rather keep your settings in a file, copy `test/database.yml.sample` to `test/database.yml` and edit it; when that file exists it is used instead (it can also use ERB).
+
 Once you have pushed up your changes, you can find your CI results [here](https://github.com/zdennis/activerecord-import/actions).
 
 #### Docker Setup
@@ -657,11 +720,15 @@ Before you begin, make sure you have [Docker](https://www.docker.com/products/do
 ```bash
 brew install docker && brew install docker-compose
 ```
-##### Steps
 
-1. In your terminal run `docker-compose up --build`
-1. In another tab/window run `docker-compose exec app bash`
-1. In that same terminal run the mysql2 test by running `bundle exec rake test:mysql2`
+`docker compose up -d` starts MySQL 8.0, MariaDB 11.8 and PostgreSQL with PostGIS (the image CI uses), listening on localhost ports 13306, 13307 and 15432 so they don't collide with databases you may already run. To use other ports, set `MYSQL_PORT`, `MARIADB_PORT` or `PG_PORT` when starting them, and the same variables (with `MYSQL_PORT` set to the MariaDB port to test MariaDB) when running the tests. `docker compose down -v` stops them and deletes their data.
+
+To run the tests inside Docker instead of with a local Ruby:
+
+1. In your terminal run `docker compose --profile app up -d --build`
+1. Then run `docker compose exec app bash`
+1. In that shell run `bundle install` (the image's gems are for `AR_VERSION=8.1`; to test another version, `rm Gemfile.lock` and run `AR_VERSION=7.2 bundle install`, then set `AR_VERSION` when running the tests too)
+1. Run the mysql2 tests with `bundle exec rake test:mysql2` (set `MYSQL_HOST=mariadb` to use MariaDB)
 
 ## Issue Triage [![Open Source Helpers](https://www.codetriage.com/zdennis/activerecord-import/badges/users.svg)](https://www.codetriage.com/zdennis/activerecord-import)
 

data/lib/activerecord-import/adapters/abstract_adapter.rb

@@ -62,6 +62,59 @@ module ActiveRecord::Import::AbstractAdapter
       post_sql_statements
     end
 
+    # Returns the selections for an INSERT ... RETURNING clause: the primary key
+    # and any +:returning+ columns. Known columns are quoted, anything else (such
+    # as raw SQL) is returned unchanged.
+    def returning_selections(options)
+      selections = []
+      column_names = Array(options[:model]&.column_names)
+
+      selections += Array(options[:primary_key]) if options[:primary_key].present?
+      selections += Array(options[:returning]) if options[:returning].present?
+
+      selections.map do |selection|
+        column_names.include?(selection.to_s) ? quote_column_name(selection) : selection
+      end
+    end
+
+    # Splits the values returned by an INSERT ... RETURNING into the ids and the
+    # +:returning+ results. +returned_values+ is a Hash with the returned
+    # +:values+, and the +:columns+ when more than one column was returned.
+    def ids_and_results_from_returned_values( returned_values, options ) # :nodoc:
+      if options[:returning].blank?
+        [Array(returned_values[:values]), []]
+      elsif options[:primary_key].blank?
+        options[:returning_columns] ||= returned_values[:columns]
+        [[], Array(returned_values[:values])]
+      else
+        # split primary key and returning columns
+        ids, results, options[:returning_columns] = split_ids_and_results(returned_values, options)
+        [ids, results]
+      end
+    end
+
+    def split_ids_and_results( selections, options )
+      ids = []
+      returning_values = []
+
+      columns = Array(selections[:columns])
+      values = Array(selections[:values])
+      id_indexes = Array(options[:primary_key]).map { |key| columns.index(key) }
+      returning_columns = columns.reject.with_index { |_, index| id_indexes.include?(index) }
+      returning_indexes = returning_columns.map { |column| columns.index(column) }
+
+      values.each do |value|
+        value_array = Array(value)
+        ids << id_indexes.map { |index| value_array[index] }
+        returning_values << returning_indexes.map { |index| value_array[index] }
+      end
+
+      ids.map!(&:first) if id_indexes.size == 1
+      returning_values.map!(&:first) if returning_columns.size == 1
+
+      [ids, returning_values, returning_columns]
+    end
+
     def increment_locking_column!(table_name, results, locking_column)
       if locking_column.present?
         results << "\"#{locking_column}\"=#{table_name}.\"#{locking_column}\"+1"
@@ -71,5 +124,10 @@ module ActiveRecord::Import::AbstractAdapter
     def supports_on_duplicate_key_update?
       false
     end
+
+    # Whether a value in an INSERT's VALUES can be DEFAULT, to use the column's default
+    def supports_default_keyword_in_values?
+      true
+    end
   end
 end

data/lib/activerecord-import/adapters/mysql_adapter.rb

@@ -6,6 +6,7 @@ module ActiveRecord::Import::MysqlAdapter
 
   NO_MAX_PACKET = 0
   QUERY_OVERHEAD = 8 # This was shown to be true for MySQL, but it's not clear where the overhead is from.
+  MIN_MARIADB_VERSION_FOR_RETURNING = "10.5.0"
 
   # +sql+ can be a single string or an array. If it is an array all
   # elements that are in position >= 1 will be appended to the final SQL.
@@ -13,6 +14,9 @@ module ActiveRecord::Import::MysqlAdapter
     # the number of inserts default
     number_of_inserts = 0
     affected_rows = nil
+    returning = import_returning?( options )
+    returned_columns = []
+    returned_rows = []
 
     base_sql, post_sql = case sql
                          when String
@@ -34,32 +38,55 @@ module ActiveRecord::Import::MysqlAdapter
 
     max = max_allowed_packet
 
-    # if we can insert it all as one statement
-    if max == NO_MAX_PACKET || total_bytes <= max || options[:force_single_insert]
+    insert_value_set = lambda do |value_set|
       number_of_inserts += 1
-      sql2insert = base_sql + values.join( ',' ) + post_sql
-      result = execute( sql2insert, *args )
-      if result.respond_to?(:affected_rows)
-        affected_rows = (affected_rows || 0) + result.affected_rows
+      sql2insert = base_sql + value_set.join( ',' ) + post_sql
+
+      if returning
+        result = exec_query( sql2insert, *args )
+        returned_columns = result.columns
+        returned_rows.concat(result.rows)
+
+        if result.respond_to?(:affected_rows) && result.affected_rows
+          affected_rows = (affected_rows || 0) + result.affected_rows
+        end
+      else
+        # Unlike #execute, whose result only has the affected rows with some drivers
+        # (e.g. trilogy, but not mysql2), #exec_update returns them with every driver
+        count = exec_update( sql2insert, *args )
+        affected_rows = (affected_rows || 0) + count if count.is_a?(Integer)
       end
+    end
+
+    # if we can insert it all as one statement
+    if max == NO_MAX_PACKET || total_bytes <= max || options[:force_single_insert]
+      insert_value_set.call(values)
     else
       value_sets = ::ActiveRecord::Import::ValueSetsBytesParser.parse(values,
         reserved_bytes: sql_size,
         max_bytes: max)
 
       transaction(requires_new: true) do
-        value_sets.each do |value_set|
-          number_of_inserts += 1
-          sql2insert = base_sql + value_set.join( ',' ) + post_sql
-          result = execute( sql2insert, *args )
-          if result.respond_to?(:affected_rows)
-            affected_rows = (affected_rows || 0) + result.affected_rows
-          end
-        end
+        value_sets.each { |value_set| insert_value_set.call(value_set) }
       end
     end
 
-    ActiveRecord::Import::Result.new([], number_of_inserts, [], [], affected_rows)
+    # ActiveRecord < 7.1 doesn't clear the query cache for #execute or #exec_query
+    clear_query_cache if query_cache_enabled
+
+    ids = []
+    results = []
+
+    if returning
+      returned_values = if returned_columns.size > 1
+        { values: returned_rows, columns: returned_columns }
+      else
+        { values: returned_rows.map(&:first) }
+      end
+      ids, results = ids_and_results_from_returned_values(returned_values, options)
+    end
+
+    ActiveRecord::Import::Result.new([], number_of_inserts, ids, results, affected_rows)
   end
 
   # Returns the maximum number of bytes that the server will allow
@@ -75,10 +102,33 @@ module ActiveRecord::Import::MysqlAdapter
 
   def pre_sql_statements( options)
     sql = []
-    sql << "IGNORE" if options[:ignore] || options[:on_duplicate_key_ignore]
+    # Recursive imports need the id of every model, which INSERT IGNORE doesn't return
+    # for ignored rows, so :ignore and :on_duplicate_key_ignore don't apply to them.
+    if (options[:ignore] || options[:on_duplicate_key_ignore]) && !(options[:recursive] && supports_setting_primary_key_of_imported_objects?)
+      sql << "IGNORE"
+    end
     sql + super
   end
 
+  def post_sql_statements( table_name, options ) # :nodoc:
+    sql = super
+    sql << "RETURNING #{returning_selections(options).join(', ')}" if import_returning?( options )
+    sql
+  end
+
+  # MariaDB 10.5+ supports INSERT ... RETURNING, so the ids of imported models
+  # can be set. MySQL doesn't support it.
+  def supports_setting_primary_key_of_imported_objects?
+    respond_to?(:mariadb?) && mariadb? && database_version >= MIN_MARIADB_VERSION_FOR_RETURNING
+  end
+
+  # Returns true if the import should use INSERT ... RETURNING
+  def import_returning?( options ) # :nodoc:
+    supports_setting_primary_key_of_imported_objects? &&
+      returning_selections(options).present? &&
+      !(options[:no_returning] && !options[:recursive])
+  end
+
   # Add a column to be updated on duplicate key update
   def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
     if (columns = options[:on_duplicate_key_update])
@@ -109,7 +159,7 @@ module ActiveRecord::Import::MysqlAdapter
 
   def sql_for_on_duplicate_key_update_as_array( table_name, model, locking_column, arr ) # :nodoc:
     results = arr.map do |column|
-      original_column_name = model.attribute_alias?( column ) ? model.attribute_alias( column ) : column
+      original_column_name = model.column_name_for_import( column )
       qc = quote_column_name( original_column_name )
       "#{table_name}.#{qc}=VALUES(#{qc})"
     end
@@ -119,10 +169,10 @@ module ActiveRecord::Import::MysqlAdapter
 
   def sql_for_on_duplicate_key_update_as_hash( table_name, model, locking_column, hsh ) # :nodoc:
     results = hsh.map do |column1, column2|
-      original_column1_name = model.attribute_alias?( column1 ) ? model.attribute_alias( column1 ) : column1
+      original_column1_name = model.column_name_for_import( column1 )
       qc1 = quote_column_name( original_column1_name )
 
-      original_column2_name = model.attribute_alias?( column2 ) ? model.attribute_alias( column2 ) : column2
+      original_column2_name = model.column_name_for_import( column2 )
       qc2 = quote_column_name( original_column2_name )
 
       "#{table_name}.#{qc1}=VALUES( #{qc2} )"