activerecord-import 1.0.2 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.travis.yml +11 -5
- data/CHANGELOG.md +67 -2
- data/Gemfile +6 -2
- data/LICENSE +21 -56
- data/README.markdown +60 -52
- data/activerecord-import.gemspec +3 -3
- data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +0 -0
- data/gemfiles/6.0.gemfile +2 -1
- data/gemfiles/6.1.gemfile +1 -1
- data/lib/activerecord-import/adapters/abstract_adapter.rb +6 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +6 -6
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +3 -9
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +7 -13
- data/lib/activerecord-import/base.rb +7 -1
- data/lib/activerecord-import/import.rb +55 -21
- data/lib/activerecord-import/value_sets_parser.rb +2 -0
- data/lib/activerecord-import/version.rb +1 -1
- data/test/import_test.rb +48 -0
- data/test/models/animal.rb +6 -0
- data/test/schema/postgresql_schema.rb +14 -0
- data/test/support/postgresql/import_examples.rb +31 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +10 -0
- data/test/support/shared_examples/recursive_import.rb +9 -0
- data/test/test_helper.rb +9 -1
- metadata +12 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4c61743fafaad0de04ccf2c6bff4439fe66aed03e951e5c42e4f7926cf8dac39
|
4
|
+
data.tar.gz: c58d8992957e546b73bb7f7b90eaf2a741f007d2109a16db0e3b3d71b376e1da
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 98638b63235eae1c16f27d3be932b8ce4e8f236b24350766792a4fc563d199cb1d048d25efa5d22abf3a21017fa07b9474c3168a4be6c1ba5aaa6e0843d584a3
|
7
|
+
data.tar.gz: 5c62bed2684f1b6d60d7e26b8ad13c6b7173a64835eeca9474082122425ab1f340f9063868224eda2539878210887701022e1ff6e6a37593872682bf90768d3a
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
language: ruby
|
2
2
|
cache: bundler
|
3
3
|
rvm:
|
4
|
-
- 2.5.
|
4
|
+
- 2.5.8
|
5
5
|
|
6
6
|
env:
|
7
7
|
global:
|
@@ -11,6 +11,7 @@ env:
|
|
11
11
|
- AR_VERSION=5.1
|
12
12
|
- AR_VERSION=5.2
|
13
13
|
- AR_VERSION=6.0
|
14
|
+
- AR_VERSION=6.1
|
14
15
|
|
15
16
|
matrix:
|
16
17
|
include:
|
@@ -28,7 +29,7 @@ matrix:
|
|
28
29
|
fast_finish: true
|
29
30
|
|
30
31
|
addons:
|
31
|
-
postgresql: "
|
32
|
+
postgresql: "10"
|
32
33
|
apt:
|
33
34
|
sources:
|
34
35
|
- travis-ci/sqlite3
|
@@ -37,13 +38,14 @@ addons:
|
|
37
38
|
- sqlite3
|
38
39
|
- mysql-server
|
39
40
|
- mysql-client
|
40
|
-
- postgresql-
|
41
|
+
- postgresql-10-postgis-2.4
|
41
42
|
|
42
43
|
before_install:
|
44
|
+
- sudo apt-get update
|
43
45
|
- gem update --system
|
44
46
|
- sudo mysql -e "use mysql; update user set authentication_string=PASSWORD('') where User='root'; update user set plugin='mysql_native_password';FLUSH PRIVILEGES;"
|
45
47
|
- sudo mysql_upgrade
|
46
|
-
- sudo service mysql restart
|
48
|
+
- sudo service mysql restart
|
47
49
|
|
48
50
|
before_script:
|
49
51
|
- mysql -e 'create database activerecord_import_test;'
|
@@ -65,6 +67,10 @@ script:
|
|
65
67
|
- bundle exec rake test:sqlite3
|
66
68
|
- bundle exec rubocop
|
67
69
|
|
68
|
-
dist:
|
70
|
+
dist: xenial
|
71
|
+
|
72
|
+
services:
|
73
|
+
- mysql
|
74
|
+
- postgresql
|
69
75
|
|
70
76
|
sudo: required
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,68 @@
|
|
1
|
+
## Changes in 1.1.0
|
2
|
+
|
3
|
+
### New Features
|
4
|
+
|
5
|
+
* Add batch progress reporting. Thanks to @gee-forr via \##729.
|
6
|
+
|
7
|
+
## Changes in 1.0.8
|
8
|
+
|
9
|
+
### Fixes
|
10
|
+
|
11
|
+
* Use correct method for clearing query cache. Thanks to @EtienneDepaulis via \##719.
|
12
|
+
|
13
|
+
## Changes in 1.0.7
|
14
|
+
|
15
|
+
### New Features
|
16
|
+
|
17
|
+
* Use @@max_allowed_packet session variable instead of querying SHOW VARIABLES. Thanks to @diclophis via \#706.
|
18
|
+
* Add option :track_validation_failures. When this is set to true, failed_instances will be an array of arrays, with each inner array having the form [:index_in_dataset, :object_with_errors]. Thanks to @rorymckinley via \#684.
|
19
|
+
|
20
|
+
### Fixes
|
21
|
+
|
22
|
+
* Prevent mass-assignment errors in Rails strict mode. Thanks to @diclophis via \##709.
|
23
|
+
|
24
|
+
## Changes in 1.0.6
|
25
|
+
|
26
|
+
### Fixes
|
27
|
+
|
28
|
+
* Handle after_initialize callbacks. Thanks to @AhMohsen46 via \#691 and
|
29
|
+
\#692.
|
30
|
+
* Fix regression introduced in 1.0.4. Explicity allow adapters to
|
31
|
+
support on duplicate key update. Thanks to @dsobiera, @jkowens via \#696.
|
32
|
+
|
33
|
+
## Changes in 1.0.5
|
34
|
+
|
35
|
+
### Fixes
|
36
|
+
|
37
|
+
* Allow serialized attributes to be returned from import. Thanks to @timanovsky, @jkowens via \#660.
|
38
|
+
* Return ActiveRecord::Connection from
|
39
|
+
ActiveREcord::Base#establish_connection. Thanks to @reverentF via
|
40
|
+
\#663.
|
41
|
+
* Support PostgreSQL array. Thanks to @ujihisa via \#669.
|
42
|
+
* Skip loading association ids when column changed. Thanks to @Aristat
|
43
|
+
via \#673.
|
44
|
+
|
45
|
+
## Changes in 1.0.4
|
46
|
+
|
47
|
+
### Fixes
|
48
|
+
|
49
|
+
* Use prepend pattern for ActiveRecord::Base#establish_connection patching. Thanks to @dombesz via \#648.
|
50
|
+
* Fix NoMethodError when using PostgreSQL ENUM types. Thanks to @sebcoetzee via \#651.
|
51
|
+
* Fix issue updating optimistic lock in Postgres. Thanks to @timanovsky
|
52
|
+
via \#656.
|
53
|
+
|
54
|
+
## Changes in 1.0.3
|
55
|
+
|
56
|
+
### New Features
|
57
|
+
|
58
|
+
* Add support for ActiveRecord 6.1.0.alpha. Thanks to @imtayadeway via
|
59
|
+
\#642.
|
60
|
+
|
61
|
+
### Fixes
|
62
|
+
|
63
|
+
* Return an empty array for results instead of nil when importing empty
|
64
|
+
array. Thanks to @gyfis via \#636.
|
65
|
+
|
1
66
|
## Changes in 1.0.2
|
2
67
|
|
3
68
|
### New Features
|
@@ -241,7 +306,7 @@
|
|
241
306
|
Thanks to @jkowens via \#301.
|
242
307
|
* Allow for custom timestamp columns. Thanks to @mojidabckuu, @jkowens
|
243
308
|
via \#401.
|
244
|
-
|
309
|
+
|
245
310
|
### Fixes
|
246
311
|
|
247
312
|
* Fix ActiveRecord 5 issue coercing boolean values when serializing
|
@@ -253,7 +318,7 @@
|
|
253
318
|
|
254
319
|
* Fix issue where PostgreSQL cannot recognize columns if names
|
255
320
|
include mixed case characters. Thanks to @hugobgranja via \#379.
|
256
|
-
* Fix an issue for ActiveRecord 5 where serialized fields with
|
321
|
+
* Fix an issue for ActiveRecord 5 where serialized fields with
|
257
322
|
default values were not being typecast. Thanks to @whistlerbrk,
|
258
323
|
@jkowens via \#386.
|
259
324
|
* Add option :force_single_insert for MySQL to make sure a single
|
data/Gemfile
CHANGED
@@ -6,8 +6,11 @@ version = ENV['AR_VERSION'].to_f
|
|
6
6
|
|
7
7
|
mysql2_version = '0.3.0'
|
8
8
|
mysql2_version = '0.4.0' if version >= 4.2
|
9
|
+
mysql2_version = '0.5.0' if version >= 6.1
|
9
10
|
sqlite3_version = '1.3.0'
|
10
|
-
sqlite3_version = '1.4.0' if version >=
|
11
|
+
sqlite3_version = '1.4.0' if version >= 6.0
|
12
|
+
pg_version = '0.9'
|
13
|
+
pg_version = '1.1' if version >= 6.1
|
11
14
|
|
12
15
|
group :development, :test do
|
13
16
|
gem 'rubocop', '~> 0.40.0'
|
@@ -17,7 +20,7 @@ end
|
|
17
20
|
# Database Adapters
|
18
21
|
platforms :ruby do
|
19
22
|
gem "mysql2", "~> #{mysql2_version}"
|
20
|
-
gem "pg", "~>
|
23
|
+
gem "pg", "~> #{pg_version}"
|
21
24
|
gem "sqlite3", "~> #{sqlite3_version}"
|
22
25
|
gem "seamless_database_pool", "~> 1.0.20"
|
23
26
|
end
|
@@ -47,6 +50,7 @@ end
|
|
47
50
|
|
48
51
|
platforms :ruby do
|
49
52
|
gem "pry-byebug"
|
53
|
+
gem "pry", "~> 0.12.0"
|
50
54
|
gem "rb-readline"
|
51
55
|
end
|
52
56
|
|
data/LICENSE
CHANGED
@@ -1,56 +1,21 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
d) make other distribution arrangements with the author.
|
24
|
-
|
25
|
-
3. You may distribute the software in object code or binary form,
|
26
|
-
provided that you do at least ONE of the following:
|
27
|
-
|
28
|
-
a) distribute the binaries and library files of the software,
|
29
|
-
together with instructions (in the manual page or equivalent)
|
30
|
-
on where to get the original distribution.
|
31
|
-
|
32
|
-
b) accompany the distribution with the machine-readable source of
|
33
|
-
the software.
|
34
|
-
|
35
|
-
c) give non-standard binaries non-standard names, with
|
36
|
-
instructions on where to get the original software distribution.
|
37
|
-
|
38
|
-
d) make other distribution arrangements with the author.
|
39
|
-
|
40
|
-
4. You may modify and include the part of the software into any other
|
41
|
-
software (possibly commercial). But some files in the distribution
|
42
|
-
are not written by the author, so that they are not under these terms.
|
43
|
-
|
44
|
-
For the list of those files and their copying conditions, see the
|
45
|
-
file LEGAL.
|
46
|
-
|
47
|
-
5. The scripts and library files supplied as input to or produced as
|
48
|
-
output from the software do not automatically fall under the
|
49
|
-
copyright of the software, but belong to whomever generated them,
|
50
|
-
and may be sold commercially, and may be aggregated with this
|
51
|
-
software.
|
52
|
-
|
53
|
-
6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
|
54
|
-
IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
|
55
|
-
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
56
|
-
PURPOSE.
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2020 Zach Dennis <zach.dennis@gmail.com>
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.markdown
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
#
|
1
|
+
# Activerecord-Import [![Build Status](https://travis-ci.org/zdennis/activerecord-import.svg?branch=master)](https://travis-ci.org/zdennis/activerecord-import)
|
2
2
|
|
3
|
-
|
3
|
+
Activerecord-Import is a library for bulk inserting data using ActiveRecord.
|
4
4
|
|
5
5
|
One of its major features is following activerecord associations and generating the minimal
|
6
6
|
number of SQL insert statements required, avoiding the N+1 insert problem. An example probably
|
@@ -23,10 +23,10 @@ an 18 hour batch process to <2 hrs.
|
|
23
23
|
|
24
24
|
The gem provides the following high-level features:
|
25
25
|
|
26
|
-
*
|
27
|
-
*
|
28
|
-
*
|
29
|
-
*
|
26
|
+
* Works with raw columns and arrays of values (fastest)
|
27
|
+
* Works with model objects (faster)
|
28
|
+
* Performs validations (fast)
|
29
|
+
* Performs on duplicate key updates (requires MySQL, SQLite 3.24.0+, or Postgres 9.5+)
|
30
30
|
|
31
31
|
## Table of Contents
|
32
32
|
|
@@ -54,11 +54,14 @@ The gem provides the following high-level features:
|
|
54
54
|
* [More Information](#more-information)
|
55
55
|
* [Contributing](#contributing)
|
56
56
|
* [Running Tests](#running-tests)
|
57
|
+
* [Issue Triage](#issue-triage)
|
57
58
|
|
58
59
|
### Examples
|
59
60
|
|
60
61
|
#### Introduction
|
61
62
|
|
63
|
+
This gem adds an `import` method (or `bulk_import`, for compatibility with gems like `elasticsearch-model`; see [Conflicts With Other Gems](#conflicts-with-other-gems)) to ActiveRecord classes.
|
64
|
+
|
62
65
|
Without `activerecord-import`, you'd write something like this:
|
63
66
|
|
64
67
|
```ruby
|
@@ -85,7 +88,7 @@ The `import` method can take an array of column names (string or symbols) and an
|
|
85
88
|
|
86
89
|
```ruby
|
87
90
|
columns = [ :title, :author ]
|
88
|
-
values = [ ['Book1', '
|
91
|
+
values = [ ['Book1', 'George Orwell'], ['Book2', 'Bob Jones'] ]
|
89
92
|
|
90
93
|
# Importing without model validations
|
91
94
|
Book.import columns, values, validate: false
|
@@ -102,7 +105,7 @@ Book.import columns, values
|
|
102
105
|
The `import` method can take an array of hashes. The keys map to the column names in the database.
|
103
106
|
|
104
107
|
```ruby
|
105
|
-
values = [{ title: 'Book1', author: '
|
108
|
+
values = [{ title: 'Book1', author: 'George Orwell' }, { title: 'Book2', author: 'Bob Jones'}]
|
106
109
|
|
107
110
|
# Importing without model validations
|
108
111
|
Book.import values, validate: false
|
@@ -119,7 +122,7 @@ The `import` method can take an array of column names and an array of hash objec
|
|
119
122
|
|
120
123
|
```ruby
|
121
124
|
books = [
|
122
|
-
{ title: "Book 1", author: "
|
125
|
+
{ title: "Book 1", author: "George Orwell" },
|
123
126
|
{ title: "Book 2", author: "Bob Jones" }
|
124
127
|
]
|
125
128
|
columns = [ :title ]
|
@@ -171,7 +174,7 @@ The `import` method can take an array of models. The attributes will be pulled o
|
|
171
174
|
|
172
175
|
```ruby
|
173
176
|
books = [
|
174
|
-
Book.new(title: "Book 1", author: "
|
177
|
+
Book.new(title: "Book 1", author: "George Orwell"),
|
175
178
|
Book.new(title: "Book 2", author: "Bob Jones")
|
176
179
|
]
|
177
180
|
|
@@ -189,7 +192,7 @@ The `import` method can take an array of column names and an array of models. Th
|
|
189
192
|
|
190
193
|
```ruby
|
191
194
|
books = [
|
192
|
-
Book.new(title: "Book 1", author: "
|
195
|
+
Book.new(title: "Book 1", author: "George Orwell"),
|
193
196
|
Book.new(title: "Book 2", author: "Bob Jones")
|
194
197
|
]
|
195
198
|
columns = [ :title ]
|
@@ -217,7 +220,7 @@ The `import` method can take a `batch_size` option to control the number of rows
|
|
217
220
|
|
218
221
|
```ruby
|
219
222
|
books = [
|
220
|
-
Book.new(title: "Book 1", author: "
|
223
|
+
Book.new(title: "Book 1", author: "George Orwell"),
|
221
224
|
Book.new(title: "Book 2", author: "Bob Jones"),
|
222
225
|
Book.new(title: "Book 1", author: "John Doe"),
|
223
226
|
Book.new(title: "Book 2", author: "Richard Wright")
|
@@ -228,9 +231,22 @@ columns = [ :title ]
|
|
228
231
|
Book.import columns, books, batch_size: 2
|
229
232
|
```
|
230
233
|
|
234
|
+
If your import is particularly large or slow (possibly due to [callbacks](#callbacks)) whilst batch importing, you might want a way to report back on progress. This is supported by passing a callable as the `batch_progress` option. e.g:
|
235
|
+
|
236
|
+
```ruby
|
237
|
+
my_proc = ->(rows_size, num_batches, current_batch_number, batch_duration_in_secs) {
|
238
|
+
# Using the arguments provided to the callable, you can
|
239
|
+
# send an email, post to a websocket,
|
240
|
+
# update slack, alert if import is taking too long, etc.
|
241
|
+
}
|
242
|
+
|
243
|
+
Book.import columns, books, batch_size: 2, batch_progress: my_proc
|
244
|
+
```
|
245
|
+
|
231
246
|
#### Recursive
|
232
247
|
|
233
|
-
NOTE: This only works with PostgreSQL.
|
248
|
+
NOTE: This only works with PostgreSQL and ActiveRecord objects. This won't work with
|
249
|
+
hashes or arrays as recursive inputs.
|
234
250
|
|
235
251
|
Assume that Books <code>has_many</code> Reviews.
|
236
252
|
|
@@ -246,19 +262,21 @@ Book.import books, recursive: true
|
|
246
262
|
|
247
263
|
### Options
|
248
264
|
|
249
|
-
Key
|
250
|
-
|
251
|
-
:validate
|
252
|
-
:validate_uniqueness
|
253
|
-
:
|
254
|
-
:
|
255
|
-
:
|
256
|
-
:
|
257
|
-
:
|
258
|
-
:
|
259
|
-
:
|
260
|
-
:
|
261
|
-
|
265
|
+
Key | Options | Default | Description
|
266
|
+
------------------------- | --------------------- | ------------------ | -----------
|
267
|
+
:validate | `true`/`false` | `true` | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
|
268
|
+
:validate_uniqueness | `true`/`false` | `false` | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
|
269
|
+
:validate_with_context | `Symbol` |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
|
270
|
+
:track_validation_failures| `true`/`false` | `false` | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]`
|
271
|
+
:on_duplicate_key_ignore | `true`/`false` | `false` | Allows skipping records with duplicate keys. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-ignore) for more details.
|
272
|
+
:ignore | `true`/`false` | `false` | Alias for :on_duplicate_key_ignore.
|
273
|
+
:on_duplicate_key_update | :all, `Array`, `Hash` | N/A | Allows upsert logic to be used. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-update) for more details.
|
274
|
+
:synchronize | `Array` | N/A | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
|
275
|
+
:timestamps | `true`/`false` | `true` | Enables/disables timestamps on imported records.
|
276
|
+
:recursive | `true`/`false` | `false` | Imports has_many/has_one associations (PostgreSQL only).
|
277
|
+
:batch_size | `Integer` | total # of records | Max number of records to insert per import
|
278
|
+
:raise_error | `true`/`false` | `false` | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
|
279
|
+
:all_or_none | `true`/`false` | `false` | Will not import any records if there is a record with validation errors.
|
262
280
|
|
263
281
|
#### Duplicate Key Ignore
|
264
282
|
|
@@ -267,14 +285,14 @@ Key | Options | Default | Descripti
|
|
267
285
|
For Postgres 9.5+ it adds `ON CONFLICT DO NOTHING`, for MySQL it uses `INSERT IGNORE`, and for SQLite it uses `INSERT OR IGNORE`. Cannot be enabled on a recursive import. For database adapters that normally support setting primary keys on imported objects, this option prevents that from occurring.
|
268
286
|
|
269
287
|
```ruby
|
270
|
-
book = Book.create! title: "Book1", author: "
|
288
|
+
book = Book.create! title: "Book1", author: "George Orwell"
|
271
289
|
book.title = "Updated Book Title"
|
272
290
|
book.author = "Bob Barker"
|
273
291
|
|
274
292
|
Book.import [book], on_duplicate_key_ignore: true
|
275
293
|
|
276
294
|
book.reload.title # => "Book1" (stayed the same)
|
277
|
-
book.reload.author # => "
|
295
|
+
book.reload.author # => "George Orwell" (stayed the same)
|
278
296
|
```
|
279
297
|
|
280
298
|
The option `:on_duplicate_key_ignore` is bypassed when `:recursive` is enabled for [PostgreSQL imports](https://github.com/zdennis/activerecord-import/wiki#recursive-example-postgresql-only).
|
@@ -290,7 +308,7 @@ This will use MySQL's `ON DUPLICATE KEY UPDATE` or Postgres/SQLite `ON CONFLICT
|
|
290
308
|
Basic Update
|
291
309
|
|
292
310
|
```ruby
|
293
|
-
book = Book.create! title: "Book1", author: "
|
311
|
+
book = Book.create! title: "Book1", author: "George Orwell"
|
294
312
|
book.title = "Updated Book Title"
|
295
313
|
book.author = "Bob Barker"
|
296
314
|
|
@@ -304,13 +322,13 @@ Book.import [book], on_duplicate_key_update: {conflict_target: [:id], columns: [
|
|
304
322
|
Book.import [book], on_duplicate_key_update: [:title]
|
305
323
|
|
306
324
|
book.reload.title # => "Updated Book Title" (changed)
|
307
|
-
book.reload.author # => "
|
325
|
+
book.reload.author # => "George Orwell" (stayed the same)
|
308
326
|
```
|
309
327
|
|
310
328
|
Using the value from another column
|
311
329
|
|
312
330
|
```ruby
|
313
|
-
book = Book.create! title: "Book1", author: "
|
331
|
+
book = Book.create! title: "Book1", author: "George Orwell"
|
314
332
|
book.title = "Updated Book Title"
|
315
333
|
|
316
334
|
# MySQL version
|
@@ -328,7 +346,7 @@ book.reload.author # => "Updated Book Title" (changed)
|
|
328
346
|
Using Custom SQL
|
329
347
|
|
330
348
|
```ruby
|
331
|
-
book = Book.create! title: "Book1", author: "
|
349
|
+
book = Book.create! title: "Book1", author: "George Orwell"
|
332
350
|
book.author = "Bob Barker"
|
333
351
|
|
334
352
|
# MySQL version
|
@@ -349,7 +367,7 @@ book.reload.author # => "Bob Barker" (changed)
|
|
349
367
|
PostgreSQL Using constraints
|
350
368
|
|
351
369
|
```ruby
|
352
|
-
book = Book.create! title: "Book1", author: "
|
370
|
+
book = Book.create! title: "Book1", author: "George Orwell", edition: 3, published_at: nil
|
353
371
|
book.published_at = Time.now
|
354
372
|
|
355
373
|
# in migration
|
@@ -363,7 +381,7 @@ Book.import [book], on_duplicate_key_update: {constraint_name: :for_upsert, colu
|
|
363
381
|
|
364
382
|
|
365
383
|
book.reload.title # => "Book1" (stayed the same)
|
366
|
-
book.reload.author # => "
|
384
|
+
book.reload.author # => "George Orwell" (stayed the same)
|
367
385
|
book.reload.edition # => 3 (stayed the same)
|
368
386
|
book.reload.published_at # => 2017-10-09 (changed)
|
369
387
|
```
|
@@ -524,7 +542,7 @@ require 'activerecord-import'
|
|
524
542
|
### Load Path Setup
|
525
543
|
To understand how rubygems loads code you can reference the following:
|
526
544
|
|
527
|
-
http://guides.rubygems.org/patterns/#
|
545
|
+
http://guides.rubygems.org/patterns/#loading-code
|
528
546
|
|
529
547
|
And an example of how active_record dynamically load adapters:
|
530
548
|
|
@@ -552,7 +570,7 @@ When rubygems pushes the `lib` folder onto the load path a `require` will now fi
|
|
552
570
|
|
553
571
|
### Conflicts With Other Gems
|
554
572
|
|
555
|
-
|
573
|
+
Activerecord-Import adds the `.import` method onto `ActiveRecord::Base`. There are other gems, such as `elasticsearch-rails`, that do the same thing. In conflicts such as this, there is an aliased method named `.bulk_import` that can be used interchangeably.
|
556
574
|
|
557
575
|
If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by explcitly setting this value within the model. For example:
|
558
576
|
|
@@ -579,7 +597,7 @@ See https://github.com/zdennis/activerecord-import/issues/233 for further discus
|
|
579
597
|
|
580
598
|
### More Information
|
581
599
|
|
582
|
-
For more information on
|
600
|
+
For more information on Activerecord-Import please see its wiki: https://github.com/zdennis/activerecord-import/wiki
|
583
601
|
|
584
602
|
To document new information, please add to the README instead of the wiki. See https://github.com/zdennis/activerecord-import/issues/397 for discussion.
|
585
603
|
|
@@ -605,24 +623,14 @@ AR_VERSION=4.2 bundle exec rake test:postgresql test:sqlite3 test:mysql2
|
|
605
623
|
|
606
624
|
Once you have pushed up your changes, you can find your CI results [here](https://travis-ci.org/zdennis/activerecord-import/).
|
607
625
|
|
626
|
+
## Issue Triage [![Open Source Helpers](https://www.codetriage.com/zdennis/activerecord-import/badges/users.svg)](https://www.codetriage.com/zdennis/activerecord-import)
|
627
|
+
|
628
|
+
You can triage issues which may include reproducing bug reports or asking for vital information, such as version numbers or reproduction instructions. If you would like to start triaging issues, one easy way to get started is to [subscribe to activerecord-import on CodeTriage](https://www.codetriage.com/zdennis/activerecord-import).
|
629
|
+
|
608
630
|
# License
|
609
631
|
|
610
|
-
This is licensed under the
|
632
|
+
This is licensed under the MIT license.
|
611
633
|
|
612
634
|
# Author
|
613
635
|
|
614
636
|
Zach Dennis (zach.dennis@gmail.com)
|
615
|
-
|
616
|
-
# Contributors
|
617
|
-
|
618
|
-
* Jordan Owens (@jkowens)
|
619
|
-
* Erik Michaels-Ober (@sferik)
|
620
|
-
* Blythe Dunham
|
621
|
-
* Gabe da Silveira
|
622
|
-
* Henry Work
|
623
|
-
* James Herdman
|
624
|
-
* Marcus Crafter
|
625
|
-
* Thibaud Guillaume-Gentil
|
626
|
-
* Mark Van Holstyn
|
627
|
-
* Victor Costan
|
628
|
-
* Dillon Welch
|
data/activerecord-import.gemspec
CHANGED
@@ -6,8 +6,8 @@ Gem::Specification.new do |gem|
|
|
6
6
|
gem.email = ["zach.dennis@gmail.com"]
|
7
7
|
gem.summary = "Bulk insert extension for ActiveRecord"
|
8
8
|
gem.description = "A library for bulk inserting data using ActiveRecord."
|
9
|
-
gem.homepage = "
|
10
|
-
gem.license = "
|
9
|
+
gem.homepage = "https://github.com/zdennis/activerecord-import"
|
10
|
+
gem.license = "MIT"
|
11
11
|
|
12
12
|
gem.files = `git ls-files`.split($\)
|
13
13
|
gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
|
@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
|
|
16
16
|
gem.require_paths = ["lib"]
|
17
17
|
gem.version = ActiveRecord::Import::VERSION
|
18
18
|
|
19
|
-
gem.required_ruby_version = ">=
|
19
|
+
gem.required_ruby_version = ">= 2.0.0"
|
20
20
|
|
21
21
|
gem.add_runtime_dependency "activerecord", ">= 3.2"
|
22
22
|
gem.add_development_dependency "rake"
|
File without changes
|
data/gemfiles/6.0.gemfile
CHANGED
@@ -1 +1,2 @@
|
|
1
|
-
gem 'activerecord', '~> 6.0.0
|
1
|
+
gem 'activerecord', '~> 6.0.0'
|
2
|
+
gem 'composite_primary_keys', '~> 12.0'
|
data/gemfiles/6.1.gemfile
CHANGED
@@ -1 +1 @@
|
|
1
|
-
gem 'activerecord', '~> 6.1.0
|
1
|
+
gem 'activerecord', '~> 6.1.0'
|
@@ -59,6 +59,12 @@ module ActiveRecord::Import::AbstractAdapter
|
|
59
59
|
post_sql_statements
|
60
60
|
end
|
61
61
|
|
62
|
+
def increment_locking_column!(table_name, results, locking_column)
|
63
|
+
if locking_column.present?
|
64
|
+
results << "\"#{locking_column}\"=#{table_name}.\"#{locking_column}\"+1"
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
62
68
|
def supports_on_duplicate_key_update?
|
63
69
|
false
|
64
70
|
end
|
@@ -56,9 +56,9 @@ module ActiveRecord::Import::MysqlAdapter
|
|
56
56
|
# in a single packet
|
57
57
|
def max_allowed_packet # :nodoc:
|
58
58
|
@max_allowed_packet ||= begin
|
59
|
-
result = execute( "
|
59
|
+
result = execute( "SELECT @@max_allowed_packet" )
|
60
60
|
# original Mysql gem responds to #fetch_row while Mysql2 responds to #first
|
61
|
-
val = result.respond_to?(:fetch_row) ? result.fetch_row[
|
61
|
+
val = result.respond_to?(:fetch_row) ? result.fetch_row[0] : result.first[0]
|
62
62
|
val.to_i
|
63
63
|
end
|
64
64
|
end
|
@@ -102,7 +102,7 @@ module ActiveRecord::Import::MysqlAdapter
|
|
102
102
|
qc = quote_column_name( column )
|
103
103
|
"#{table_name}.#{qc}=VALUES(#{qc})"
|
104
104
|
end
|
105
|
-
increment_locking_column!(
|
105
|
+
increment_locking_column!(table_name, results, locking_column)
|
106
106
|
results.join( ',' )
|
107
107
|
end
|
108
108
|
|
@@ -112,7 +112,7 @@ module ActiveRecord::Import::MysqlAdapter
|
|
112
112
|
qc2 = quote_column_name( column2 )
|
113
113
|
"#{table_name}.#{qc1}=VALUES( #{qc2} )"
|
114
114
|
end
|
115
|
-
increment_locking_column!(
|
115
|
+
increment_locking_column!(table_name, results, locking_column)
|
116
116
|
results.join( ',')
|
117
117
|
end
|
118
118
|
|
@@ -121,9 +121,9 @@ module ActiveRecord::Import::MysqlAdapter
|
|
121
121
|
exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry')
|
122
122
|
end
|
123
123
|
|
124
|
-
def increment_locking_column!(
|
124
|
+
def increment_locking_column!(table_name, results, locking_column)
|
125
125
|
if locking_column.present?
|
126
|
-
results << "
|
126
|
+
results << "`#{locking_column}`=#{table_name}.`#{locking_column}`+1"
|
127
127
|
end
|
128
128
|
end
|
129
129
|
end
|
@@ -28,7 +28,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
28
28
|
else
|
29
29
|
select_values( sql2insert, *args )
|
30
30
|
end
|
31
|
-
|
31
|
+
clear_query_cache if query_cache_enabled
|
32
32
|
end
|
33
33
|
|
34
34
|
if options[:returning].blank?
|
@@ -158,7 +158,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
158
158
|
qc = quote_column_name( column )
|
159
159
|
"#{qc}=EXCLUDED.#{qc}"
|
160
160
|
end
|
161
|
-
increment_locking_column!(results, locking_column)
|
161
|
+
increment_locking_column!(table_name, results, locking_column)
|
162
162
|
results.join( ',' )
|
163
163
|
end
|
164
164
|
|
@@ -168,7 +168,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
168
168
|
qc2 = quote_column_name( column2 )
|
169
169
|
"#{qc1}=EXCLUDED.#{qc2}"
|
170
170
|
end
|
171
|
-
increment_locking_column!(results, locking_column)
|
171
|
+
increment_locking_column!(table_name, results, locking_column)
|
172
172
|
results.join( ',' )
|
173
173
|
end
|
174
174
|
|
@@ -203,12 +203,6 @@ module ActiveRecord::Import::PostgreSQLAdapter
|
|
203
203
|
true
|
204
204
|
end
|
205
205
|
|
206
|
-
def increment_locking_column!(results, locking_column)
|
207
|
-
if locking_column.present?
|
208
|
-
results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
|
209
|
-
end
|
210
|
-
end
|
211
|
-
|
212
206
|
private
|
213
207
|
|
214
208
|
def database_version
|
@@ -92,7 +92,7 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
92
92
|
|
93
93
|
# Returns a generated ON CONFLICT DO UPDATE statement given the passed
|
94
94
|
# in +args+.
|
95
|
-
def sql_for_on_duplicate_key_update(
|
95
|
+
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
96
96
|
arg, primary_key, locking_column = args
|
97
97
|
arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
|
98
98
|
return unless arg.is_a?( Hash )
|
@@ -113,9 +113,9 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
113
113
|
|
114
114
|
sql << "#{conflict_target}DO UPDATE SET "
|
115
115
|
if columns.is_a?( Array )
|
116
|
-
sql << sql_for_on_duplicate_key_update_as_array( locking_column, columns )
|
116
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, columns )
|
117
117
|
elsif columns.is_a?( Hash )
|
118
|
-
sql << sql_for_on_duplicate_key_update_as_hash( locking_column, columns )
|
118
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, columns )
|
119
119
|
elsif columns.is_a?( String )
|
120
120
|
sql << columns
|
121
121
|
else
|
@@ -127,22 +127,22 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
127
127
|
sql
|
128
128
|
end
|
129
129
|
|
130
|
-
def sql_for_on_duplicate_key_update_as_array( locking_column, arr ) # :nodoc:
|
130
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
|
131
131
|
results = arr.map do |column|
|
132
132
|
qc = quote_column_name( column )
|
133
133
|
"#{qc}=EXCLUDED.#{qc}"
|
134
134
|
end
|
135
|
-
increment_locking_column!(results, locking_column)
|
135
|
+
increment_locking_column!(table_name, results, locking_column)
|
136
136
|
results.join( ',' )
|
137
137
|
end
|
138
138
|
|
139
|
-
def sql_for_on_duplicate_key_update_as_hash( locking_column, hsh ) # :nodoc:
|
139
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
|
140
140
|
results = hsh.map do |column1, column2|
|
141
141
|
qc1 = quote_column_name( column1 )
|
142
142
|
qc2 = quote_column_name( column2 )
|
143
143
|
"#{qc1}=EXCLUDED.#{qc2}"
|
144
144
|
end
|
145
|
-
increment_locking_column!(results, locking_column)
|
145
|
+
increment_locking_column!(table_name, results, locking_column)
|
146
146
|
results.join( ',' )
|
147
147
|
end
|
148
148
|
|
@@ -166,12 +166,6 @@ module ActiveRecord::Import::SQLite3Adapter
|
|
166
166
|
exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
|
167
167
|
end
|
168
168
|
|
169
|
-
def increment_locking_column!(results, locking_column)
|
170
|
-
if locking_column.present?
|
171
|
-
results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
|
172
|
-
end
|
173
|
-
end
|
174
|
-
|
175
169
|
private
|
176
170
|
|
177
171
|
def database_version
|
@@ -27,7 +27,13 @@ module ActiveRecord::Import
|
|
27
27
|
|
28
28
|
# Loads the import functionality for the passed in ActiveRecord connection
|
29
29
|
def self.load_from_connection_pool(connection_pool)
|
30
|
-
|
30
|
+
adapter =
|
31
|
+
if connection_pool.respond_to?(:db_config) # ActiveRecord >= 6.1
|
32
|
+
connection_pool.db_config.adapter
|
33
|
+
else
|
34
|
+
connection_pool.spec.config[:adapter]
|
35
|
+
end
|
36
|
+
require_adapter adapter
|
31
37
|
end
|
32
38
|
end
|
33
39
|
|
@@ -245,16 +245,17 @@ class ActiveRecord::Associations::CollectionAssociation
|
|
245
245
|
alias import bulk_import unless respond_to? :import
|
246
246
|
end
|
247
247
|
|
248
|
+
module ActiveRecord::Import::Connection
|
249
|
+
def establish_connection(args = nil)
|
250
|
+
conn = super(args)
|
251
|
+
ActiveRecord::Import.load_from_connection_pool connection_pool
|
252
|
+
conn
|
253
|
+
end
|
254
|
+
end
|
255
|
+
|
248
256
|
class ActiveRecord::Base
|
249
257
|
class << self
|
250
|
-
|
251
|
-
conn = establish_connection_without_activerecord_import(*args)
|
252
|
-
ActiveRecord::Import.load_from_connection_pool connection_pool
|
253
|
-
conn
|
254
|
-
end
|
255
|
-
|
256
|
-
alias establish_connection_without_activerecord_import establish_connection
|
257
|
-
alias establish_connection establish_connection_with_activerecord_import
|
258
|
+
prepend ActiveRecord::Import::Connection
|
258
259
|
|
259
260
|
# Returns true if the current database connection adapter
|
260
261
|
# supports import functionality, otherwise returns false.
|
@@ -546,7 +547,7 @@ class ActiveRecord::Base
|
|
546
547
|
alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
|
547
548
|
|
548
549
|
def import_helper( *args )
|
549
|
-
options = { validate: true, timestamps: true }
|
550
|
+
options = { validate: true, timestamps: true, track_validation_failures: false }
|
550
551
|
options.merge!( args.pop ) if args.last.is_a? Hash
|
551
552
|
# making sure that current model's primary key is used
|
552
553
|
options[:primary_key] = primary_key
|
@@ -581,7 +582,7 @@ class ActiveRecord::Base
|
|
581
582
|
if respond_to?(:timestamp_attributes_for_update, true)
|
582
583
|
send(:timestamp_attributes_for_update).map(&:to_sym)
|
583
584
|
else
|
584
|
-
|
585
|
+
allocate.send(:timestamp_attributes_for_update_in_model)
|
585
586
|
end
|
586
587
|
end
|
587
588
|
|
@@ -630,7 +631,7 @@ class ActiveRecord::Base
|
|
630
631
|
end
|
631
632
|
# supports empty array
|
632
633
|
elsif args.last.is_a?( Array ) && args.last.empty?
|
633
|
-
return ActiveRecord::Import::Result.new([], 0, [])
|
634
|
+
return ActiveRecord::Import::Result.new([], 0, [], [])
|
634
635
|
# supports 2-element array and array
|
635
636
|
elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
|
636
637
|
|
@@ -702,14 +703,18 @@ class ActiveRecord::Base
|
|
702
703
|
# keep track of the instance and the position it is currently at. if this fails
|
703
704
|
# validation we'll use the index to remove it from the array_of_attributes
|
704
705
|
arr.each_with_index do |hsh, i|
|
705
|
-
|
706
|
-
|
706
|
+
# utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
|
707
|
+
model = new do |m|
|
708
|
+
hsh.each_pair { |k, v| m[k] = v }
|
709
|
+
end
|
710
|
+
|
707
711
|
next if validator.valid_model?(model)
|
708
712
|
raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
|
713
|
+
|
709
714
|
array_of_attributes[i] = nil
|
710
715
|
failure = model.dup
|
711
716
|
failure.errors.send(:initialize_dup, model.errors)
|
712
|
-
failed_instances << failure
|
717
|
+
failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
|
713
718
|
end
|
714
719
|
array_of_attributes.compact!
|
715
720
|
end
|
@@ -800,17 +805,29 @@ class ActiveRecord::Base
|
|
800
805
|
if supports_import?
|
801
806
|
# generate the sql
|
802
807
|
post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
|
808
|
+
import_size = values_sql.size
|
809
|
+
|
810
|
+
batch_size = options[:batch_size] || import_size
|
811
|
+
run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
|
812
|
+
progress_proc = options[:batch_progress]
|
813
|
+
current_batch = 0
|
814
|
+
batches = (import_size / batch_size.to_f).ceil
|
803
815
|
|
804
|
-
batch_size = options[:batch_size] || values_sql.size
|
805
816
|
values_sql.each_slice(batch_size) do |batch_values|
|
817
|
+
batch_started_at = Time.now.to_i
|
818
|
+
|
806
819
|
# perform the inserts
|
807
820
|
result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
|
808
821
|
batch_values,
|
809
822
|
options,
|
810
|
-
"#{model_name} Create Many
|
823
|
+
"#{model_name} Create Many" )
|
824
|
+
|
811
825
|
number_inserted += result.num_inserts
|
812
826
|
ids += result.ids
|
813
827
|
results += result.results
|
828
|
+
current_batch += 1
|
829
|
+
|
830
|
+
progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
|
814
831
|
end
|
815
832
|
else
|
816
833
|
transaction(requires_new: true) do
|
@@ -841,6 +858,19 @@ class ActiveRecord::Base
|
|
841
858
|
end
|
842
859
|
end
|
843
860
|
|
861
|
+
deserialize_value = lambda do |column, value|
|
862
|
+
column = columns_hash[column]
|
863
|
+
return value unless column
|
864
|
+
if respond_to?(:type_caster)
|
865
|
+
type = type_for_attribute(column.name)
|
866
|
+
type.deserialize(value)
|
867
|
+
elsif column.respond_to?(:type_cast_from_database)
|
868
|
+
column.type_cast_from_database(value)
|
869
|
+
else
|
870
|
+
value
|
871
|
+
end
|
872
|
+
end
|
873
|
+
|
844
874
|
if models.size == import_result.results.size
|
845
875
|
columns = Array(options[:returning])
|
846
876
|
single_column = "#{columns.first}=" if columns.size == 1
|
@@ -848,10 +878,12 @@ class ActiveRecord::Base
|
|
848
878
|
model = models[index]
|
849
879
|
|
850
880
|
if single_column
|
851
|
-
|
881
|
+
val = deserialize_value.call(columns.first, result)
|
882
|
+
model.send(single_column, val)
|
852
883
|
else
|
853
884
|
columns.each_with_index do |column, col_index|
|
854
|
-
|
885
|
+
val = deserialize_value.call(column, result[col_index])
|
886
|
+
model.send("#{column}=", val)
|
855
887
|
end
|
856
888
|
end
|
857
889
|
end
|
@@ -872,10 +904,12 @@ class ActiveRecord::Base
|
|
872
904
|
|
873
905
|
# Sync belongs_to association ids with foreign key field
|
874
906
|
def load_association_ids(model)
|
907
|
+
changed_columns = model.changed
|
875
908
|
association_reflections = model.class.reflect_on_all_associations(:belongs_to)
|
876
909
|
association_reflections.each do |association_reflection|
|
877
910
|
column_name = association_reflection.foreign_key
|
878
911
|
next if association_reflection.options[:polymorphic]
|
912
|
+
next if changed_columns.include?(column_name)
|
879
913
|
association = model.association(association_reflection.name)
|
880
914
|
association = association.target
|
881
915
|
next if association.blank? || model.public_send(column_name).present?
|
@@ -955,7 +989,7 @@ class ActiveRecord::Base
|
|
955
989
|
elsif column
|
956
990
|
if respond_to?(:type_caster) # Rails 5.0 and higher
|
957
991
|
type = type_for_attribute(column.name)
|
958
|
-
val = type.type == :boolean ? type.cast(val) : type.serialize(val)
|
992
|
+
val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
|
959
993
|
connection_memo.quote(val)
|
960
994
|
elsif column.respond_to?(:type_cast_from_user) # Rails 4.2
|
961
995
|
connection_memo.quote(column.type_cast_from_user(val), column)
|
@@ -964,7 +998,7 @@ class ActiveRecord::Base
|
|
964
998
|
val = serialized_attributes[column.name].dump(val)
|
965
999
|
end
|
966
1000
|
# Fixes #443 to support binary (i.e. bytea) columns on PG
|
967
|
-
val = column.type_cast(val) unless column.type.to_sym == :binary
|
1001
|
+
val = column.type_cast(val) unless column.type && column.type.to_sym == :binary
|
968
1002
|
connection_memo.quote(val, column)
|
969
1003
|
end
|
970
1004
|
else
|
@@ -983,7 +1017,7 @@ class ActiveRecord::Base
|
|
983
1017
|
timestamp_columns[:create] = timestamp_attributes_for_create_in_model
|
984
1018
|
timestamp_columns[:update] = timestamp_attributes_for_update_in_model
|
985
1019
|
else
|
986
|
-
instance =
|
1020
|
+
instance = allocate
|
987
1021
|
timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
|
988
1022
|
timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
|
989
1023
|
end
|
data/test/import_test.rb
CHANGED
@@ -252,6 +252,16 @@ describe "#import" do
|
|
252
252
|
end
|
253
253
|
end
|
254
254
|
|
255
|
+
it "should index the failed instances by their poistion in the set if `track_failures` is true" do
|
256
|
+
index_offset = valid_values.length
|
257
|
+
results = Topic.import columns, valid_values + invalid_values, validate: true, track_validation_failures: true
|
258
|
+
assert_equal invalid_values.size, results.failed_instances.size
|
259
|
+
invalid_values.each_with_index do |value_set, index|
|
260
|
+
assert_equal index + index_offset, results.failed_instances[index].first
|
261
|
+
assert_equal value_set.first, results.failed_instances[index].last.title
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
255
265
|
it "should set ids in valid models if adapter supports setting primary key of imported objects" do
|
256
266
|
if ActiveRecord::Base.supports_setting_primary_key_of_imported_objects?
|
257
267
|
Topic.import (invalid_models + valid_models), validate: true
|
@@ -395,6 +405,15 @@ describe "#import" do
|
|
395
405
|
assert_equal 3, result.num_inserts if Topic.supports_import?
|
396
406
|
end
|
397
407
|
end
|
408
|
+
|
409
|
+
it "should accept and call an optional callable to run after each batch" do
|
410
|
+
lambda_called = 0
|
411
|
+
|
412
|
+
my_proc = ->(_row_count, _batches, _batch, _duration) { lambda_called += 1 }
|
413
|
+
Topic.import Build(10, :topics), batch_size: 4, batch_progress: my_proc
|
414
|
+
|
415
|
+
assert_equal 3, lambda_called
|
416
|
+
end
|
398
417
|
end
|
399
418
|
|
400
419
|
context "with :synchronize option" do
|
@@ -900,4 +919,33 @@ describe "#import" do
|
|
900
919
|
end
|
901
920
|
end
|
902
921
|
end
|
922
|
+
describe "importing model with after_initialize callback" do
|
923
|
+
let(:columns) { %w(name size) }
|
924
|
+
let(:valid_values) { [%w("Deer", "Small"), %w("Monkey", "Medium")] }
|
925
|
+
let(:invalid_values) do
|
926
|
+
[
|
927
|
+
{ name: "giraffe", size: "Large" },
|
928
|
+
{ size: "Medium" } # name is missing
|
929
|
+
]
|
930
|
+
end
|
931
|
+
context "with validation checks turned off" do
|
932
|
+
it "should import valid data" do
|
933
|
+
Animal.import(columns, valid_values, validate: false)
|
934
|
+
assert_equal 2, Animal.count
|
935
|
+
end
|
936
|
+
it "should raise ArgumentError" do
|
937
|
+
assert_raise(ArgumentError) { Animal.import(invalid_values, validate: false) }
|
938
|
+
end
|
939
|
+
end
|
940
|
+
|
941
|
+
context "with validation checks turned on" do
|
942
|
+
it "should import valid data" do
|
943
|
+
Animal.import(columns, valid_values, validate: true)
|
944
|
+
assert_equal 2, Animal.count
|
945
|
+
end
|
946
|
+
it "should raise ArgumentError" do
|
947
|
+
assert_raise(ArgumentError) { Animal.import(invalid_values, validate: true) }
|
948
|
+
end
|
949
|
+
end
|
950
|
+
end
|
903
951
|
end
|
@@ -3,8 +3,20 @@ ActiveRecord::Schema.define do
|
|
3
3
|
execute('CREATE extension IF NOT EXISTS "pgcrypto";')
|
4
4
|
execute('CREATE extension IF NOT EXISTS "uuid-ossp";')
|
5
5
|
|
6
|
+
# create ENUM if it does not exist yet
|
7
|
+
begin
|
8
|
+
execute('CREATE TYPE vendor_type AS ENUM (\'wholesaler\', \'retailer\');')
|
9
|
+
rescue ActiveRecord::StatementInvalid => e
|
10
|
+
# since PostgreSQL does not support IF NOT EXISTS when creating a TYPE,
|
11
|
+
# rescue the error and check the error class
|
12
|
+
raise unless e.cause.is_a? PG::DuplicateObject
|
13
|
+
execute('ALTER TYPE vendor_type ADD VALUE IF NOT EXISTS \'wholesaler\';')
|
14
|
+
execute('ALTER TYPE vendor_type ADD VALUE IF NOT EXISTS \'retailer\';')
|
15
|
+
end
|
16
|
+
|
6
17
|
create_table :vendors, id: :uuid, force: :cascade do |t|
|
7
18
|
t.string :name, null: true
|
19
|
+
t.text :hours
|
8
20
|
t.text :preferences
|
9
21
|
|
10
22
|
if t.respond_to?(:json)
|
@@ -29,6 +41,8 @@ ActiveRecord::Schema.define do
|
|
29
41
|
t.text :json_data
|
30
42
|
end
|
31
43
|
|
44
|
+
t.column :vendor_type, :vendor_type
|
45
|
+
|
32
46
|
t.datetime :created_at
|
33
47
|
t.datetime :updated_at
|
34
48
|
end
|
@@ -116,6 +116,26 @@ def should_support_postgresql_import_functionality
|
|
116
116
|
assert_equal [%w(King It)], result.results
|
117
117
|
end
|
118
118
|
|
119
|
+
context "when given an empty array" do
|
120
|
+
let(:result) { Book.import([], returning: %w(title)) }
|
121
|
+
|
122
|
+
setup { result }
|
123
|
+
|
124
|
+
it "returns empty arrays for ids and results" do
|
125
|
+
assert_equal [], result.ids
|
126
|
+
assert_equal [], result.results
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
context "when a returning column is a serialized attribute" do
|
131
|
+
let(:vendor) { Vendor.new(hours: { monday: '8-5' }) }
|
132
|
+
let(:result) { Vendor.import([vendor], returning: %w(hours)) }
|
133
|
+
|
134
|
+
it "creates records" do
|
135
|
+
assert_difference("Vendor.count", +1) { result }
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
119
139
|
context "when primary key and returning overlap" do
|
120
140
|
let(:result) { Book.import(books, returning: %w(id title)) }
|
121
141
|
|
@@ -249,6 +269,17 @@ def should_support_postgresql_import_functionality
|
|
249
269
|
end
|
250
270
|
end
|
251
271
|
|
272
|
+
describe "with enum field" do
|
273
|
+
let(:vendor_type) { "retailer" }
|
274
|
+
it "imports the correct values for enum fields" do
|
275
|
+
vendor = Vendor.new(name: 'Vendor 1', vendor_type: vendor_type)
|
276
|
+
assert_difference "Vendor.count", +1 do
|
277
|
+
Vendor.import [vendor]
|
278
|
+
end
|
279
|
+
assert_equal(vendor_type, Vendor.first.vendor_type)
|
280
|
+
end
|
281
|
+
end
|
282
|
+
|
252
283
|
describe "with binary field" do
|
253
284
|
let(:binary_value) { "\xE0'c\xB2\xB0\xB3Bh\\\xC2M\xB1m\\I\xC4r".force_encoding('ASCII-8BIT') }
|
254
285
|
it "imports the correct values for binary fields" do
|
@@ -73,6 +73,16 @@ def should_support_basic_on_duplicate_key_update
|
|
73
73
|
assert_equal user.name, users[i].name + ' Rothschild'
|
74
74
|
assert_equal 1, user.lock_version
|
75
75
|
end
|
76
|
+
updated_values2 = User.all.map do |user|
|
77
|
+
user.name += ' jr.'
|
78
|
+
{ id: user.id, name: user.name }
|
79
|
+
end
|
80
|
+
User.import(updated_values2, on_duplicate_key_update: [:name])
|
81
|
+
assert User.count == updated_values2.length
|
82
|
+
User.all.each_with_index do |user, i|
|
83
|
+
assert_equal user.name, users[i].name + ' Rothschild jr.'
|
84
|
+
assert_equal 2, user.lock_version
|
85
|
+
end
|
76
86
|
end
|
77
87
|
|
78
88
|
it 'upsert optimistic lock columns other than lock_version by model' do
|
@@ -138,6 +138,15 @@ def should_support_recursive_import
|
|
138
138
|
books.each do |book|
|
139
139
|
assert_equal book.topic_id, second_new_topic.id
|
140
140
|
end
|
141
|
+
|
142
|
+
books.each { |book| book.topic_id = nil }
|
143
|
+
assert_no_difference "Book.count", books.size do
|
144
|
+
Book.import books, validate: false, on_duplicate_key_update: [:topic_id]
|
145
|
+
end
|
146
|
+
|
147
|
+
books.each do |book|
|
148
|
+
assert_equal book.topic_id, nil
|
149
|
+
end
|
141
150
|
end
|
142
151
|
|
143
152
|
unless ENV["SKIP_COMPOSITE_PK"]
|
data/test/test_helper.rb
CHANGED
@@ -48,7 +48,15 @@ adapter = ENV["ARE_DB"] || "sqlite3"
|
|
48
48
|
FileUtils.mkdir_p 'log'
|
49
49
|
ActiveRecord::Base.logger = Logger.new("log/test.log")
|
50
50
|
ActiveRecord::Base.logger.level = Logger::DEBUG
|
51
|
-
|
51
|
+
|
52
|
+
if ENV['AR_VERSION'].to_f >= 6.0
|
53
|
+
yaml_config = YAML.load_file(test_dir.join("database.yml"))[adapter]
|
54
|
+
config = ActiveRecord::DatabaseConfigurations::HashConfig.new("test", adapter, yaml_config)
|
55
|
+
ActiveRecord::Base.configurations.configurations << config
|
56
|
+
else
|
57
|
+
ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
|
58
|
+
end
|
59
|
+
|
52
60
|
ActiveRecord::Base.default_timezone = :utc
|
53
61
|
|
54
62
|
require "activerecord-import"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: activerecord-import
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Zach Dennis
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-05-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -67,7 +67,7 @@ files:
|
|
67
67
|
- benchmarks/models/test_innodb.rb
|
68
68
|
- benchmarks/models/test_memory.rb
|
69
69
|
- benchmarks/models/test_myisam.rb
|
70
|
-
- benchmarks/schema/
|
70
|
+
- benchmarks/schema/mysql2_schema.rb
|
71
71
|
- gemfiles/3.2.gemfile
|
72
72
|
- gemfiles/4.0.gemfile
|
73
73
|
- gemfiles/4.1.gemfile
|
@@ -121,6 +121,7 @@ files:
|
|
121
121
|
- test/makara_postgis/import_test.rb
|
122
122
|
- test/models/account.rb
|
123
123
|
- test/models/alarm.rb
|
124
|
+
- test/models/animal.rb
|
124
125
|
- test/models/bike_maker.rb
|
125
126
|
- test/models/book.rb
|
126
127
|
- test/models/car.rb
|
@@ -166,11 +167,11 @@ files:
|
|
166
167
|
- test/travis/database.yml
|
167
168
|
- test/value_sets_bytes_parser_test.rb
|
168
169
|
- test/value_sets_records_parser_test.rb
|
169
|
-
homepage:
|
170
|
+
homepage: https://github.com/zdennis/activerecord-import
|
170
171
|
licenses:
|
171
|
-
-
|
172
|
+
- MIT
|
172
173
|
metadata: {}
|
173
|
-
post_install_message:
|
174
|
+
post_install_message:
|
174
175
|
rdoc_options: []
|
175
176
|
require_paths:
|
176
177
|
- lib
|
@@ -178,16 +179,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
178
179
|
requirements:
|
179
180
|
- - ">="
|
180
181
|
- !ruby/object:Gem::Version
|
181
|
-
version:
|
182
|
+
version: 2.0.0
|
182
183
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
183
184
|
requirements:
|
184
185
|
- - ">="
|
185
186
|
- !ruby/object:Gem::Version
|
186
187
|
version: '0'
|
187
188
|
requirements: []
|
188
|
-
|
189
|
-
|
190
|
-
signing_key:
|
189
|
+
rubygems_version: 3.0.8
|
190
|
+
signing_key:
|
191
191
|
specification_version: 4
|
192
192
|
summary: Bulk insert extension for ActiveRecord
|
193
193
|
test_files:
|
@@ -212,6 +212,7 @@ test_files:
|
|
212
212
|
- test/makara_postgis/import_test.rb
|
213
213
|
- test/models/account.rb
|
214
214
|
- test/models/alarm.rb
|
215
|
+
- test/models/animal.rb
|
215
216
|
- test/models/bike_maker.rb
|
216
217
|
- test/models/book.rb
|
217
218
|
- test/models/car.rb
|