activerecord-import 1.0.2 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5448c6493533f39965a0b3cd0f00e3d2811e7f8feaa2f806e20b47d8db504c27
4
- data.tar.gz: fde9e0e991022d01d339da3da9b89d0523771533fc684d3d955af91b718bede2
3
+ metadata.gz: 4c61743fafaad0de04ccf2c6bff4439fe66aed03e951e5c42e4f7926cf8dac39
4
+ data.tar.gz: c58d8992957e546b73bb7f7b90eaf2a741f007d2109a16db0e3b3d71b376e1da
5
5
  SHA512:
6
- metadata.gz: 1048e6bb250eb8cedb71d37dad9e13245799638e948f83016e3301ea933418327fe294438ab862d0cf1c521da51fabd8ac648f16798d1f44a62cb4784176a60a
7
- data.tar.gz: e764901dc217df8fb650a788064fcd274d7141e0a3e201ecc9011aa0ce5ce89ff7bb06d899543e084543bf52e54d8feb81c6a17d464e3a81b7df5ee187c2614d
6
+ metadata.gz: 98638b63235eae1c16f27d3be932b8ce4e8f236b24350766792a4fc563d199cb1d048d25efa5d22abf3a21017fa07b9474c3168a4be6c1ba5aaa6e0843d584a3
7
+ data.tar.gz: 5c62bed2684f1b6d60d7e26b8ad13c6b7173a64835eeca9474082122425ab1f340f9063868224eda2539878210887701022e1ff6e6a37593872682bf90768d3a
data/.gitignore CHANGED
@@ -24,6 +24,7 @@ pkg
24
24
  log/*.log
25
25
  test.db
26
26
  test/database.yml
27
+ benchmarks/log/
27
28
 
28
29
  .ruby-*
29
30
  .bundle/
data/.travis.yml CHANGED
@@ -1,7 +1,7 @@
1
1
  language: ruby
2
2
  cache: bundler
3
3
  rvm:
4
- - 2.5.5
4
+ - 2.5.8
5
5
 
6
6
  env:
7
7
  global:
@@ -11,6 +11,7 @@ env:
11
11
  - AR_VERSION=5.1
12
12
  - AR_VERSION=5.2
13
13
  - AR_VERSION=6.0
14
+ - AR_VERSION=6.1
14
15
 
15
16
  matrix:
16
17
  include:
@@ -28,7 +29,7 @@ matrix:
28
29
  fast_finish: true
29
30
 
30
31
  addons:
31
- postgresql: "9.5"
32
+ postgresql: "10"
32
33
  apt:
33
34
  sources:
34
35
  - travis-ci/sqlite3
@@ -37,13 +38,14 @@ addons:
37
38
  - sqlite3
38
39
  - mysql-server
39
40
  - mysql-client
40
- - postgresql-9.5-postgis-2.3
41
+ - postgresql-10-postgis-2.4
41
42
 
42
43
  before_install:
44
+ - sudo apt-get update
43
45
  - gem update --system
44
46
  - sudo mysql -e "use mysql; update user set authentication_string=PASSWORD('') where User='root'; update user set plugin='mysql_native_password';FLUSH PRIVILEGES;"
45
47
  - sudo mysql_upgrade
46
- - sudo service mysql restart
48
+ - sudo service mysql restart
47
49
 
48
50
  before_script:
49
51
  - mysql -e 'create database activerecord_import_test;'
@@ -65,6 +67,10 @@ script:
65
67
  - bundle exec rake test:sqlite3
66
68
  - bundle exec rubocop
67
69
 
68
- dist: trusty
70
+ dist: xenial
71
+
72
+ services:
73
+ - mysql
74
+ - postgresql
69
75
 
70
76
  sudo: required
data/CHANGELOG.md CHANGED
@@ -1,3 +1,68 @@
1
+ ## Changes in 1.1.0
2
+
3
+ ### New Features
4
+
5
+ * Add batch progress reporting. Thanks to @gee-forr via \##729.
6
+
7
+ ## Changes in 1.0.8
8
+
9
+ ### Fixes
10
+
11
+ * Use correct method for clearing query cache. Thanks to @EtienneDepaulis via \##719.
12
+
13
+ ## Changes in 1.0.7
14
+
15
+ ### New Features
16
+
17
+ * Use @@max_allowed_packet session variable instead of querying SHOW VARIABLES. Thanks to @diclophis via \#706.
18
+ * Add option :track_validation_failures. When this is set to true, failed_instances will be an array of arrays, with each inner array having the form [:index_in_dataset, :object_with_errors]. Thanks to @rorymckinley via \#684.
19
+
20
+ ### Fixes
21
+
22
+ * Prevent mass-assignment errors in Rails strict mode. Thanks to @diclophis via \##709.
23
+
24
+ ## Changes in 1.0.6
25
+
26
+ ### Fixes
27
+
28
+ * Handle after_initialize callbacks. Thanks to @AhMohsen46 via \#691 and
29
+ \#692.
30
+ * Fix regression introduced in 1.0.4. Explicity allow adapters to
31
+ support on duplicate key update. Thanks to @dsobiera, @jkowens via \#696.
32
+
33
+ ## Changes in 1.0.5
34
+
35
+ ### Fixes
36
+
37
+ * Allow serialized attributes to be returned from import. Thanks to @timanovsky, @jkowens via \#660.
38
+ * Return ActiveRecord::Connection from
39
+ ActiveREcord::Base#establish_connection. Thanks to @reverentF via
40
+ \#663.
41
+ * Support PostgreSQL array. Thanks to @ujihisa via \#669.
42
+ * Skip loading association ids when column changed. Thanks to @Aristat
43
+ via \#673.
44
+
45
+ ## Changes in 1.0.4
46
+
47
+ ### Fixes
48
+
49
+ * Use prepend pattern for ActiveRecord::Base#establish_connection patching. Thanks to @dombesz via \#648.
50
+ * Fix NoMethodError when using PostgreSQL ENUM types. Thanks to @sebcoetzee via \#651.
51
+ * Fix issue updating optimistic lock in Postgres. Thanks to @timanovsky
52
+ via \#656.
53
+
54
+ ## Changes in 1.0.3
55
+
56
+ ### New Features
57
+
58
+ * Add support for ActiveRecord 6.1.0.alpha. Thanks to @imtayadeway via
59
+ \#642.
60
+
61
+ ### Fixes
62
+
63
+ * Return an empty array for results instead of nil when importing empty
64
+ array. Thanks to @gyfis via \#636.
65
+
1
66
  ## Changes in 1.0.2
2
67
 
3
68
  ### New Features
@@ -241,7 +306,7 @@
241
306
  Thanks to @jkowens via \#301.
242
307
  * Allow for custom timestamp columns. Thanks to @mojidabckuu, @jkowens
243
308
  via \#401.
244
-
309
+
245
310
  ### Fixes
246
311
 
247
312
  * Fix ActiveRecord 5 issue coercing boolean values when serializing
@@ -253,7 +318,7 @@
253
318
 
254
319
  * Fix issue where PostgreSQL cannot recognize columns if names
255
320
  include mixed case characters. Thanks to @hugobgranja via \#379.
256
- * Fix an issue for ActiveRecord 5 where serialized fields with
321
+ * Fix an issue for ActiveRecord 5 where serialized fields with
257
322
  default values were not being typecast. Thanks to @whistlerbrk,
258
323
  @jkowens via \#386.
259
324
  * Add option :force_single_insert for MySQL to make sure a single
data/Gemfile CHANGED
@@ -6,8 +6,11 @@ version = ENV['AR_VERSION'].to_f
6
6
 
7
7
  mysql2_version = '0.3.0'
8
8
  mysql2_version = '0.4.0' if version >= 4.2
9
+ mysql2_version = '0.5.0' if version >= 6.1
9
10
  sqlite3_version = '1.3.0'
10
- sqlite3_version = '1.4.0' if version >= 5.1
11
+ sqlite3_version = '1.4.0' if version >= 6.0
12
+ pg_version = '0.9'
13
+ pg_version = '1.1' if version >= 6.1
11
14
 
12
15
  group :development, :test do
13
16
  gem 'rubocop', '~> 0.40.0'
@@ -17,7 +20,7 @@ end
17
20
  # Database Adapters
18
21
  platforms :ruby do
19
22
  gem "mysql2", "~> #{mysql2_version}"
20
- gem "pg", "~> 0.9"
23
+ gem "pg", "~> #{pg_version}"
21
24
  gem "sqlite3", "~> #{sqlite3_version}"
22
25
  gem "seamless_database_pool", "~> 1.0.20"
23
26
  end
@@ -47,6 +50,7 @@ end
47
50
 
48
51
  platforms :ruby do
49
52
  gem "pry-byebug"
53
+ gem "pry", "~> 0.12.0"
50
54
  gem "rb-readline"
51
55
  end
52
56
 
data/LICENSE CHANGED
@@ -1,56 +1,21 @@
1
- Ruby is copyrighted free software by Yukihiro Matsumoto <matz@netlab.jp>.
2
- You can redistribute it and/or modify it under either the terms of the
3
- 2-clause BSDL (see the file BSDL), or the conditions below:
4
-
5
- 1. You may make and give away verbatim copies of the source form of the
6
- software without restriction, provided that you duplicate all of the
7
- original copyright notices and associated disclaimers.
8
-
9
- 2. You may modify your copy of the software in any way, provided that
10
- you do at least ONE of the following:
11
-
12
- a) place your modifications in the Public Domain or otherwise
13
- make them Freely Available, such as by posting said
14
- modifications to Usenet or an equivalent medium, or by allowing
15
- the author to include your modifications in the software.
16
-
17
- b) use the modified software only within your corporation or
18
- organization.
19
-
20
- c) give non-standard binaries non-standard names, with
21
- instructions on where to get the original software distribution.
22
-
23
- d) make other distribution arrangements with the author.
24
-
25
- 3. You may distribute the software in object code or binary form,
26
- provided that you do at least ONE of the following:
27
-
28
- a) distribute the binaries and library files of the software,
29
- together with instructions (in the manual page or equivalent)
30
- on where to get the original distribution.
31
-
32
- b) accompany the distribution with the machine-readable source of
33
- the software.
34
-
35
- c) give non-standard binaries non-standard names, with
36
- instructions on where to get the original software distribution.
37
-
38
- d) make other distribution arrangements with the author.
39
-
40
- 4. You may modify and include the part of the software into any other
41
- software (possibly commercial). But some files in the distribution
42
- are not written by the author, so that they are not under these terms.
43
-
44
- For the list of those files and their copying conditions, see the
45
- file LEGAL.
46
-
47
- 5. The scripts and library files supplied as input to or produced as
48
- output from the software do not automatically fall under the
49
- copyright of the software, but belong to whomever generated them,
50
- and may be sold commercially, and may be aggregated with this
51
- software.
52
-
53
- 6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
54
- IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
55
- WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
56
- PURPOSE.
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2020 Zach Dennis <zach.dennis@gmail.com>
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.markdown CHANGED
@@ -1,6 +1,6 @@
1
- # activerecord-import [![Build Status](https://travis-ci.org/zdennis/activerecord-import.svg?branch=master)](https://travis-ci.org/zdennis/activerecord-import)
1
+ # Activerecord-Import [![Build Status](https://travis-ci.org/zdennis/activerecord-import.svg?branch=master)](https://travis-ci.org/zdennis/activerecord-import)
2
2
 
3
- activerecord-import is a library for bulk inserting data using ActiveRecord.
3
+ Activerecord-Import is a library for bulk inserting data using ActiveRecord.
4
4
 
5
5
  One of its major features is following activerecord associations and generating the minimal
6
6
  number of SQL insert statements required, avoiding the N+1 insert problem. An example probably
@@ -23,10 +23,10 @@ an 18 hour batch process to <2 hrs.
23
23
 
24
24
  The gem provides the following high-level features:
25
25
 
26
- * activerecord-import can work with raw columns and arrays of values (fastest)
27
- * activerecord-import works with model objects (faster)
28
- * activerecord-import can perform validations (fast)
29
- * activerecord-import can perform on duplicate key updates (requires MySQL or Postgres 9.5+)
26
+ * Works with raw columns and arrays of values (fastest)
27
+ * Works with model objects (faster)
28
+ * Performs validations (fast)
29
+ * Performs on duplicate key updates (requires MySQL, SQLite 3.24.0+, or Postgres 9.5+)
30
30
 
31
31
  ## Table of Contents
32
32
 
@@ -54,11 +54,14 @@ The gem provides the following high-level features:
54
54
  * [More Information](#more-information)
55
55
  * [Contributing](#contributing)
56
56
  * [Running Tests](#running-tests)
57
+ * [Issue Triage](#issue-triage)
57
58
 
58
59
  ### Examples
59
60
 
60
61
  #### Introduction
61
62
 
63
+ This gem adds an `import` method (or `bulk_import`, for compatibility with gems like `elasticsearch-model`; see [Conflicts With Other Gems](#conflicts-with-other-gems)) to ActiveRecord classes.
64
+
62
65
  Without `activerecord-import`, you'd write something like this:
63
66
 
64
67
  ```ruby
@@ -85,7 +88,7 @@ The `import` method can take an array of column names (string or symbols) and an
85
88
 
86
89
  ```ruby
87
90
  columns = [ :title, :author ]
88
- values = [ ['Book1', 'FooManChu'], ['Book2', 'Bob Jones'] ]
91
+ values = [ ['Book1', 'George Orwell'], ['Book2', 'Bob Jones'] ]
89
92
 
90
93
  # Importing without model validations
91
94
  Book.import columns, values, validate: false
@@ -102,7 +105,7 @@ Book.import columns, values
102
105
  The `import` method can take an array of hashes. The keys map to the column names in the database.
103
106
 
104
107
  ```ruby
105
- values = [{ title: 'Book1', author: 'FooManChu' }, { title: 'Book2', author: 'Bob Jones'}]
108
+ values = [{ title: 'Book1', author: 'George Orwell' }, { title: 'Book2', author: 'Bob Jones'}]
106
109
 
107
110
  # Importing without model validations
108
111
  Book.import values, validate: false
@@ -119,7 +122,7 @@ The `import` method can take an array of column names and an array of hash objec
119
122
 
120
123
  ```ruby
121
124
  books = [
122
- { title: "Book 1", author: "FooManChu" },
125
+ { title: "Book 1", author: "George Orwell" },
123
126
  { title: "Book 2", author: "Bob Jones" }
124
127
  ]
125
128
  columns = [ :title ]
@@ -171,7 +174,7 @@ The `import` method can take an array of models. The attributes will be pulled o
171
174
 
172
175
  ```ruby
173
176
  books = [
174
- Book.new(title: "Book 1", author: "FooManChu"),
177
+ Book.new(title: "Book 1", author: "George Orwell"),
175
178
  Book.new(title: "Book 2", author: "Bob Jones")
176
179
  ]
177
180
 
@@ -189,7 +192,7 @@ The `import` method can take an array of column names and an array of models. Th
189
192
 
190
193
  ```ruby
191
194
  books = [
192
- Book.new(title: "Book 1", author: "FooManChu"),
195
+ Book.new(title: "Book 1", author: "George Orwell"),
193
196
  Book.new(title: "Book 2", author: "Bob Jones")
194
197
  ]
195
198
  columns = [ :title ]
@@ -217,7 +220,7 @@ The `import` method can take a `batch_size` option to control the number of rows
217
220
 
218
221
  ```ruby
219
222
  books = [
220
- Book.new(title: "Book 1", author: "FooManChu"),
223
+ Book.new(title: "Book 1", author: "George Orwell"),
221
224
  Book.new(title: "Book 2", author: "Bob Jones"),
222
225
  Book.new(title: "Book 1", author: "John Doe"),
223
226
  Book.new(title: "Book 2", author: "Richard Wright")
@@ -228,9 +231,22 @@ columns = [ :title ]
228
231
  Book.import columns, books, batch_size: 2
229
232
  ```
230
233
 
234
+ If your import is particularly large or slow (possibly due to [callbacks](#callbacks)) whilst batch importing, you might want a way to report back on progress. This is supported by passing a callable as the `batch_progress` option. e.g:
235
+
236
+ ```ruby
237
+ my_proc = ->(rows_size, num_batches, current_batch_number, batch_duration_in_secs) {
238
+ # Using the arguments provided to the callable, you can
239
+ # send an email, post to a websocket,
240
+ # update slack, alert if import is taking too long, etc.
241
+ }
242
+
243
+ Book.import columns, books, batch_size: 2, batch_progress: my_proc
244
+ ```
245
+
231
246
  #### Recursive
232
247
 
233
- NOTE: This only works with PostgreSQL.
248
+ NOTE: This only works with PostgreSQL and ActiveRecord objects. This won't work with
249
+ hashes or arrays as recursive inputs.
234
250
 
235
251
  Assume that Books <code>has_many</code> Reviews.
236
252
 
@@ -246,19 +262,21 @@ Book.import books, recursive: true
246
262
 
247
263
  ### Options
248
264
 
249
- Key | Options | Default | Description
250
- ----------------------- | --------------------- | ------------------ | -----------
251
- :validate | `true`/`false` | `true` | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
252
- :validate_uniqueness | `true`/`false` | `false` | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
253
- :on_duplicate_key_ignore| `true`/`false` | `false` | Allows skipping records with duplicate keys. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-ignore) for more details.
254
- :ignore | `true`/`false` | `false` | Alias for :on_duplicate_key_ignore.
255
- :on_duplicate_key_update| :all, `Array`, `Hash` | N/A | Allows upsert logic to be used. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-update) for more details.
256
- :synchronize | `Array` | N/A | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
257
- :timestamps | `true`/`false` | `true` | Enables/disables timestamps on imported records.
258
- :recursive | `true`/`false` | `false` | Imports has_many/has_one associations (PostgreSQL only).
259
- :batch_size | `Integer` | total # of records | Max number of records to insert per import
260
- :raise_error | `true`/`false` | `false` | Throws an exception if there are invalid records. `import!` is a shortcut for this.
261
-
265
+ Key | Options | Default | Description
266
+ ------------------------- | --------------------- | ------------------ | -----------
267
+ :validate | `true`/`false` | `true` | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
268
+ :validate_uniqueness | `true`/`false` | `false` | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
269
+ :validate_with_context | `Symbol` |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
270
+ :track_validation_failures| `true`/`false` | `false` | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]`
271
+ :on_duplicate_key_ignore | `true`/`false` | `false` | Allows skipping records with duplicate keys. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-ignore) for more details.
272
+ :ignore | `true`/`false` | `false` | Alias for :on_duplicate_key_ignore.
273
+ :on_duplicate_key_update | :all, `Array`, `Hash` | N/A | Allows upsert logic to be used. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-update) for more details.
274
+ :synchronize | `Array` | N/A | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
275
+ :timestamps | `true`/`false` | `true` | Enables/disables timestamps on imported records.
276
+ :recursive | `true`/`false` | `false` | Imports has_many/has_one associations (PostgreSQL only).
277
+ :batch_size | `Integer` | total # of records | Max number of records to insert per import
278
+ :raise_error | `true`/`false` | `false` | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
279
+ :all_or_none | `true`/`false` | `false` | Will not import any records if there is a record with validation errors.
262
280
 
263
281
  #### Duplicate Key Ignore
264
282
 
@@ -267,14 +285,14 @@ Key | Options | Default | Descripti
267
285
  For Postgres 9.5+ it adds `ON CONFLICT DO NOTHING`, for MySQL it uses `INSERT IGNORE`, and for SQLite it uses `INSERT OR IGNORE`. Cannot be enabled on a recursive import. For database adapters that normally support setting primary keys on imported objects, this option prevents that from occurring.
268
286
 
269
287
  ```ruby
270
- book = Book.create! title: "Book1", author: "FooManChu"
288
+ book = Book.create! title: "Book1", author: "George Orwell"
271
289
  book.title = "Updated Book Title"
272
290
  book.author = "Bob Barker"
273
291
 
274
292
  Book.import [book], on_duplicate_key_ignore: true
275
293
 
276
294
  book.reload.title # => "Book1" (stayed the same)
277
- book.reload.author # => "FooManChu" (stayed the same)
295
+ book.reload.author # => "George Orwell" (stayed the same)
278
296
  ```
279
297
 
280
298
  The option `:on_duplicate_key_ignore` is bypassed when `:recursive` is enabled for [PostgreSQL imports](https://github.com/zdennis/activerecord-import/wiki#recursive-example-postgresql-only).
@@ -290,7 +308,7 @@ This will use MySQL's `ON DUPLICATE KEY UPDATE` or Postgres/SQLite `ON CONFLICT
290
308
  Basic Update
291
309
 
292
310
  ```ruby
293
- book = Book.create! title: "Book1", author: "FooManChu"
311
+ book = Book.create! title: "Book1", author: "George Orwell"
294
312
  book.title = "Updated Book Title"
295
313
  book.author = "Bob Barker"
296
314
 
@@ -304,13 +322,13 @@ Book.import [book], on_duplicate_key_update: {conflict_target: [:id], columns: [
304
322
  Book.import [book], on_duplicate_key_update: [:title]
305
323
 
306
324
  book.reload.title # => "Updated Book Title" (changed)
307
- book.reload.author # => "FooManChu" (stayed the same)
325
+ book.reload.author # => "George Orwell" (stayed the same)
308
326
  ```
309
327
 
310
328
  Using the value from another column
311
329
 
312
330
  ```ruby
313
- book = Book.create! title: "Book1", author: "FooManChu"
331
+ book = Book.create! title: "Book1", author: "George Orwell"
314
332
  book.title = "Updated Book Title"
315
333
 
316
334
  # MySQL version
@@ -328,7 +346,7 @@ book.reload.author # => "Updated Book Title" (changed)
328
346
  Using Custom SQL
329
347
 
330
348
  ```ruby
331
- book = Book.create! title: "Book1", author: "FooManChu"
349
+ book = Book.create! title: "Book1", author: "George Orwell"
332
350
  book.author = "Bob Barker"
333
351
 
334
352
  # MySQL version
@@ -349,7 +367,7 @@ book.reload.author # => "Bob Barker" (changed)
349
367
  PostgreSQL Using constraints
350
368
 
351
369
  ```ruby
352
- book = Book.create! title: "Book1", author: "FooManChu", edition: 3, published_at: nil
370
+ book = Book.create! title: "Book1", author: "George Orwell", edition: 3, published_at: nil
353
371
  book.published_at = Time.now
354
372
 
355
373
  # in migration
@@ -363,7 +381,7 @@ Book.import [book], on_duplicate_key_update: {constraint_name: :for_upsert, colu
363
381
 
364
382
 
365
383
  book.reload.title # => "Book1" (stayed the same)
366
- book.reload.author # => "FooManChu" (stayed the same)
384
+ book.reload.author # => "George Orwell" (stayed the same)
367
385
  book.reload.edition # => 3 (stayed the same)
368
386
  book.reload.published_at # => 2017-10-09 (changed)
369
387
  ```
@@ -524,7 +542,7 @@ require 'activerecord-import'
524
542
  ### Load Path Setup
525
543
  To understand how rubygems loads code you can reference the following:
526
544
 
527
- http://guides.rubygems.org/patterns/#loading_code
545
+ http://guides.rubygems.org/patterns/#loading-code
528
546
 
529
547
  And an example of how active_record dynamically load adapters:
530
548
 
@@ -552,7 +570,7 @@ When rubygems pushes the `lib` folder onto the load path a `require` will now fi
552
570
 
553
571
  ### Conflicts With Other Gems
554
572
 
555
- `activerecord-import` adds the `.import` method onto `ActiveRecord::Base`. There are other gems, such as `elasticsearch-rails`, that do the same thing. In conflicts such as this, there is an aliased method named `.bulk_import` that can be used interchangeably.
573
+ Activerecord-Import adds the `.import` method onto `ActiveRecord::Base`. There are other gems, such as `elasticsearch-rails`, that do the same thing. In conflicts such as this, there is an aliased method named `.bulk_import` that can be used interchangeably.
556
574
 
557
575
  If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by explcitly setting this value within the model. For example:
558
576
 
@@ -579,7 +597,7 @@ See https://github.com/zdennis/activerecord-import/issues/233 for further discus
579
597
 
580
598
  ### More Information
581
599
 
582
- For more information on activerecord-import please see its wiki: https://github.com/zdennis/activerecord-import/wiki
600
+ For more information on Activerecord-Import please see its wiki: https://github.com/zdennis/activerecord-import/wiki
583
601
 
584
602
  To document new information, please add to the README instead of the wiki. See https://github.com/zdennis/activerecord-import/issues/397 for discussion.
585
603
 
@@ -605,24 +623,14 @@ AR_VERSION=4.2 bundle exec rake test:postgresql test:sqlite3 test:mysql2
605
623
 
606
624
  Once you have pushed up your changes, you can find your CI results [here](https://travis-ci.org/zdennis/activerecord-import/).
607
625
 
626
+ ## Issue Triage [![Open Source Helpers](https://www.codetriage.com/zdennis/activerecord-import/badges/users.svg)](https://www.codetriage.com/zdennis/activerecord-import)
627
+
628
+ You can triage issues which may include reproducing bug reports or asking for vital information, such as version numbers or reproduction instructions. If you would like to start triaging issues, one easy way to get started is to [subscribe to activerecord-import on CodeTriage](https://www.codetriage.com/zdennis/activerecord-import).
629
+
608
630
  # License
609
631
 
610
- This is licensed under the ruby license.
632
+ This is licensed under the MIT license.
611
633
 
612
634
  # Author
613
635
 
614
636
  Zach Dennis (zach.dennis@gmail.com)
615
-
616
- # Contributors
617
-
618
- * Jordan Owens (@jkowens)
619
- * Erik Michaels-Ober (@sferik)
620
- * Blythe Dunham
621
- * Gabe da Silveira
622
- * Henry Work
623
- * James Herdman
624
- * Marcus Crafter
625
- * Thibaud Guillaume-Gentil
626
- * Mark Van Holstyn
627
- * Victor Costan
628
- * Dillon Welch
@@ -6,8 +6,8 @@ Gem::Specification.new do |gem|
6
6
  gem.email = ["zach.dennis@gmail.com"]
7
7
  gem.summary = "Bulk insert extension for ActiveRecord"
8
8
  gem.description = "A library for bulk inserting data using ActiveRecord."
9
- gem.homepage = "http://github.com/zdennis/activerecord-import"
10
- gem.license = "Ruby"
9
+ gem.homepage = "https://github.com/zdennis/activerecord-import"
10
+ gem.license = "MIT"
11
11
 
12
12
  gem.files = `git ls-files`.split($\)
13
13
  gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
16
16
  gem.require_paths = ["lib"]
17
17
  gem.version = ActiveRecord::Import::VERSION
18
18
 
19
- gem.required_ruby_version = ">= 1.9.2"
19
+ gem.required_ruby_version = ">= 2.0.0"
20
20
 
21
21
  gem.add_runtime_dependency "activerecord", ">= 3.2"
22
22
  gem.add_development_dependency "rake"
data/gemfiles/6.0.gemfile CHANGED
@@ -1 +1,2 @@
1
- gem 'activerecord', '~> 6.0.0.rc1'
1
+ gem 'activerecord', '~> 6.0.0'
2
+ gem 'composite_primary_keys', '~> 12.0'
data/gemfiles/6.1.gemfile CHANGED
@@ -1 +1 @@
1
- gem 'activerecord', '~> 6.1.0.alpha'
1
+ gem 'activerecord', '~> 6.1.0'
@@ -59,6 +59,12 @@ module ActiveRecord::Import::AbstractAdapter
59
59
  post_sql_statements
60
60
  end
61
61
 
62
+ def increment_locking_column!(table_name, results, locking_column)
63
+ if locking_column.present?
64
+ results << "\"#{locking_column}\"=#{table_name}.\"#{locking_column}\"+1"
65
+ end
66
+ end
67
+
62
68
  def supports_on_duplicate_key_update?
63
69
  false
64
70
  end
@@ -56,9 +56,9 @@ module ActiveRecord::Import::MysqlAdapter
56
56
  # in a single packet
57
57
  def max_allowed_packet # :nodoc:
58
58
  @max_allowed_packet ||= begin
59
- result = execute( "SHOW VARIABLES like 'max_allowed_packet'" )
59
+ result = execute( "SELECT @@max_allowed_packet" )
60
60
  # original Mysql gem responds to #fetch_row while Mysql2 responds to #first
61
- val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1]
61
+ val = result.respond_to?(:fetch_row) ? result.fetch_row[0] : result.first[0]
62
62
  val.to_i
63
63
  end
64
64
  end
@@ -102,7 +102,7 @@ module ActiveRecord::Import::MysqlAdapter
102
102
  qc = quote_column_name( column )
103
103
  "#{table_name}.#{qc}=VALUES(#{qc})"
104
104
  end
105
- increment_locking_column!(results, table_name, locking_column)
105
+ increment_locking_column!(table_name, results, locking_column)
106
106
  results.join( ',' )
107
107
  end
108
108
 
@@ -112,7 +112,7 @@ module ActiveRecord::Import::MysqlAdapter
112
112
  qc2 = quote_column_name( column2 )
113
113
  "#{table_name}.#{qc1}=VALUES( #{qc2} )"
114
114
  end
115
- increment_locking_column!(results, table_name, locking_column)
115
+ increment_locking_column!(table_name, results, locking_column)
116
116
  results.join( ',')
117
117
  end
118
118
 
@@ -121,9 +121,9 @@ module ActiveRecord::Import::MysqlAdapter
121
121
  exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry')
122
122
  end
123
123
 
124
- def increment_locking_column!(results, table_name, locking_column)
124
+ def increment_locking_column!(table_name, results, locking_column)
125
125
  if locking_column.present?
126
- results << "#{table_name}.`#{locking_column}`=`#{locking_column}`+1"
126
+ results << "`#{locking_column}`=#{table_name}.`#{locking_column}`+1"
127
127
  end
128
128
  end
129
129
  end
@@ -28,7 +28,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
28
28
  else
29
29
  select_values( sql2insert, *args )
30
30
  end
31
- query_cache.clear if query_cache_enabled
31
+ clear_query_cache if query_cache_enabled
32
32
  end
33
33
 
34
34
  if options[:returning].blank?
@@ -158,7 +158,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
158
158
  qc = quote_column_name( column )
159
159
  "#{qc}=EXCLUDED.#{qc}"
160
160
  end
161
- increment_locking_column!(results, locking_column)
161
+ increment_locking_column!(table_name, results, locking_column)
162
162
  results.join( ',' )
163
163
  end
164
164
 
@@ -168,7 +168,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
168
168
  qc2 = quote_column_name( column2 )
169
169
  "#{qc1}=EXCLUDED.#{qc2}"
170
170
  end
171
- increment_locking_column!(results, locking_column)
171
+ increment_locking_column!(table_name, results, locking_column)
172
172
  results.join( ',' )
173
173
  end
174
174
 
@@ -203,12 +203,6 @@ module ActiveRecord::Import::PostgreSQLAdapter
203
203
  true
204
204
  end
205
205
 
206
- def increment_locking_column!(results, locking_column)
207
- if locking_column.present?
208
- results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
209
- end
210
- end
211
-
212
206
  private
213
207
 
214
208
  def database_version
@@ -92,7 +92,7 @@ module ActiveRecord::Import::SQLite3Adapter
92
92
 
93
93
  # Returns a generated ON CONFLICT DO UPDATE statement given the passed
94
94
  # in +args+.
95
- def sql_for_on_duplicate_key_update( _table_name, *args ) # :nodoc:
95
+ def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
96
96
  arg, primary_key, locking_column = args
97
97
  arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
98
98
  return unless arg.is_a?( Hash )
@@ -113,9 +113,9 @@ module ActiveRecord::Import::SQLite3Adapter
113
113
 
114
114
  sql << "#{conflict_target}DO UPDATE SET "
115
115
  if columns.is_a?( Array )
116
- sql << sql_for_on_duplicate_key_update_as_array( locking_column, columns )
116
+ sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, columns )
117
117
  elsif columns.is_a?( Hash )
118
- sql << sql_for_on_duplicate_key_update_as_hash( locking_column, columns )
118
+ sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, columns )
119
119
  elsif columns.is_a?( String )
120
120
  sql << columns
121
121
  else
@@ -127,22 +127,22 @@ module ActiveRecord::Import::SQLite3Adapter
127
127
  sql
128
128
  end
129
129
 
130
- def sql_for_on_duplicate_key_update_as_array( locking_column, arr ) # :nodoc:
130
+ def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
131
131
  results = arr.map do |column|
132
132
  qc = quote_column_name( column )
133
133
  "#{qc}=EXCLUDED.#{qc}"
134
134
  end
135
- increment_locking_column!(results, locking_column)
135
+ increment_locking_column!(table_name, results, locking_column)
136
136
  results.join( ',' )
137
137
  end
138
138
 
139
- def sql_for_on_duplicate_key_update_as_hash( locking_column, hsh ) # :nodoc:
139
+ def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
140
140
  results = hsh.map do |column1, column2|
141
141
  qc1 = quote_column_name( column1 )
142
142
  qc2 = quote_column_name( column2 )
143
143
  "#{qc1}=EXCLUDED.#{qc2}"
144
144
  end
145
- increment_locking_column!(results, locking_column)
145
+ increment_locking_column!(table_name, results, locking_column)
146
146
  results.join( ',' )
147
147
  end
148
148
 
@@ -166,12 +166,6 @@ module ActiveRecord::Import::SQLite3Adapter
166
166
  exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
167
167
  end
168
168
 
169
- def increment_locking_column!(results, locking_column)
170
- if locking_column.present?
171
- results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
172
- end
173
- end
174
-
175
169
  private
176
170
 
177
171
  def database_version
@@ -27,7 +27,13 @@ module ActiveRecord::Import
27
27
 
28
28
  # Loads the import functionality for the passed in ActiveRecord connection
29
29
  def self.load_from_connection_pool(connection_pool)
30
- require_adapter connection_pool.spec.config[:adapter]
30
+ adapter =
31
+ if connection_pool.respond_to?(:db_config) # ActiveRecord >= 6.1
32
+ connection_pool.db_config.adapter
33
+ else
34
+ connection_pool.spec.config[:adapter]
35
+ end
36
+ require_adapter adapter
31
37
  end
32
38
  end
33
39
 
@@ -245,16 +245,17 @@ class ActiveRecord::Associations::CollectionAssociation
245
245
  alias import bulk_import unless respond_to? :import
246
246
  end
247
247
 
248
+ module ActiveRecord::Import::Connection
249
+ def establish_connection(args = nil)
250
+ conn = super(args)
251
+ ActiveRecord::Import.load_from_connection_pool connection_pool
252
+ conn
253
+ end
254
+ end
255
+
248
256
  class ActiveRecord::Base
249
257
  class << self
250
- def establish_connection_with_activerecord_import(*args)
251
- conn = establish_connection_without_activerecord_import(*args)
252
- ActiveRecord::Import.load_from_connection_pool connection_pool
253
- conn
254
- end
255
-
256
- alias establish_connection_without_activerecord_import establish_connection
257
- alias establish_connection establish_connection_with_activerecord_import
258
+ prepend ActiveRecord::Import::Connection
258
259
 
259
260
  # Returns true if the current database connection adapter
260
261
  # supports import functionality, otherwise returns false.
@@ -546,7 +547,7 @@ class ActiveRecord::Base
546
547
  alias import! bulk_import! unless ActiveRecord::Base.respond_to? :import!
547
548
 
548
549
  def import_helper( *args )
549
- options = { validate: true, timestamps: true }
550
+ options = { validate: true, timestamps: true, track_validation_failures: false }
550
551
  options.merge!( args.pop ) if args.last.is_a? Hash
551
552
  # making sure that current model's primary key is used
552
553
  options[:primary_key] = primary_key
@@ -581,7 +582,7 @@ class ActiveRecord::Base
581
582
  if respond_to?(:timestamp_attributes_for_update, true)
582
583
  send(:timestamp_attributes_for_update).map(&:to_sym)
583
584
  else
584
- new.send(:timestamp_attributes_for_update_in_model)
585
+ allocate.send(:timestamp_attributes_for_update_in_model)
585
586
  end
586
587
  end
587
588
 
@@ -630,7 +631,7 @@ class ActiveRecord::Base
630
631
  end
631
632
  # supports empty array
632
633
  elsif args.last.is_a?( Array ) && args.last.empty?
633
- return ActiveRecord::Import::Result.new([], 0, [])
634
+ return ActiveRecord::Import::Result.new([], 0, [], [])
634
635
  # supports 2-element array and array
635
636
  elsif args.size == 2 && args.first.is_a?( Array ) && args.last.is_a?( Array )
636
637
 
@@ -702,14 +703,18 @@ class ActiveRecord::Base
702
703
  # keep track of the instance and the position it is currently at. if this fails
703
704
  # validation we'll use the index to remove it from the array_of_attributes
704
705
  arr.each_with_index do |hsh, i|
705
- model = new
706
- hsh.each_pair { |k, v| model[k] = v }
706
+ # utilize block initializer syntax to prevent failure when 'mass_assignment_sanitizer = :strict'
707
+ model = new do |m|
708
+ hsh.each_pair { |k, v| m[k] = v }
709
+ end
710
+
707
711
  next if validator.valid_model?(model)
708
712
  raise(ActiveRecord::RecordInvalid, model) if options[:raise_error]
713
+
709
714
  array_of_attributes[i] = nil
710
715
  failure = model.dup
711
716
  failure.errors.send(:initialize_dup, model.errors)
712
- failed_instances << failure
717
+ failed_instances << (options[:track_validation_failures] ? [i, failure] : failure )
713
718
  end
714
719
  array_of_attributes.compact!
715
720
  end
@@ -800,17 +805,29 @@ class ActiveRecord::Base
800
805
  if supports_import?
801
806
  # generate the sql
802
807
  post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
808
+ import_size = values_sql.size
809
+
810
+ batch_size = options[:batch_size] || import_size
811
+ run_proc = options[:batch_size].to_i.positive? && options[:batch_progress].respond_to?( :call )
812
+ progress_proc = options[:batch_progress]
813
+ current_batch = 0
814
+ batches = (import_size / batch_size.to_f).ceil
803
815
 
804
- batch_size = options[:batch_size] || values_sql.size
805
816
  values_sql.each_slice(batch_size) do |batch_values|
817
+ batch_started_at = Time.now.to_i
818
+
806
819
  # perform the inserts
807
820
  result = connection.insert_many( [insert_sql, post_sql_statements].flatten,
808
821
  batch_values,
809
822
  options,
810
- "#{model_name} Create Many Without Validations Or Callbacks" )
823
+ "#{model_name} Create Many" )
824
+
811
825
  number_inserted += result.num_inserts
812
826
  ids += result.ids
813
827
  results += result.results
828
+ current_batch += 1
829
+
830
+ progress_proc.call(import_size, batches, current_batch, Time.now.to_i - batch_started_at) if run_proc
814
831
  end
815
832
  else
816
833
  transaction(requires_new: true) do
@@ -841,6 +858,19 @@ class ActiveRecord::Base
841
858
  end
842
859
  end
843
860
 
861
+ deserialize_value = lambda do |column, value|
862
+ column = columns_hash[column]
863
+ return value unless column
864
+ if respond_to?(:type_caster)
865
+ type = type_for_attribute(column.name)
866
+ type.deserialize(value)
867
+ elsif column.respond_to?(:type_cast_from_database)
868
+ column.type_cast_from_database(value)
869
+ else
870
+ value
871
+ end
872
+ end
873
+
844
874
  if models.size == import_result.results.size
845
875
  columns = Array(options[:returning])
846
876
  single_column = "#{columns.first}=" if columns.size == 1
@@ -848,10 +878,12 @@ class ActiveRecord::Base
848
878
  model = models[index]
849
879
 
850
880
  if single_column
851
- model.send(single_column, result)
881
+ val = deserialize_value.call(columns.first, result)
882
+ model.send(single_column, val)
852
883
  else
853
884
  columns.each_with_index do |column, col_index|
854
- model.send("#{column}=", result[col_index])
885
+ val = deserialize_value.call(column, result[col_index])
886
+ model.send("#{column}=", val)
855
887
  end
856
888
  end
857
889
  end
@@ -872,10 +904,12 @@ class ActiveRecord::Base
872
904
 
873
905
  # Sync belongs_to association ids with foreign key field
874
906
  def load_association_ids(model)
907
+ changed_columns = model.changed
875
908
  association_reflections = model.class.reflect_on_all_associations(:belongs_to)
876
909
  association_reflections.each do |association_reflection|
877
910
  column_name = association_reflection.foreign_key
878
911
  next if association_reflection.options[:polymorphic]
912
+ next if changed_columns.include?(column_name)
879
913
  association = model.association(association_reflection.name)
880
914
  association = association.target
881
915
  next if association.blank? || model.public_send(column_name).present?
@@ -955,7 +989,7 @@ class ActiveRecord::Base
955
989
  elsif column
956
990
  if respond_to?(:type_caster) # Rails 5.0 and higher
957
991
  type = type_for_attribute(column.name)
958
- val = type.type == :boolean ? type.cast(val) : type.serialize(val)
992
+ val = !type.respond_to?(:subtype) && type.type == :boolean ? type.cast(val) : type.serialize(val)
959
993
  connection_memo.quote(val)
960
994
  elsif column.respond_to?(:type_cast_from_user) # Rails 4.2
961
995
  connection_memo.quote(column.type_cast_from_user(val), column)
@@ -964,7 +998,7 @@ class ActiveRecord::Base
964
998
  val = serialized_attributes[column.name].dump(val)
965
999
  end
966
1000
  # Fixes #443 to support binary (i.e. bytea) columns on PG
967
- val = column.type_cast(val) unless column.type.to_sym == :binary
1001
+ val = column.type_cast(val) unless column.type && column.type.to_sym == :binary
968
1002
  connection_memo.quote(val, column)
969
1003
  end
970
1004
  else
@@ -983,7 +1017,7 @@ class ActiveRecord::Base
983
1017
  timestamp_columns[:create] = timestamp_attributes_for_create_in_model
984
1018
  timestamp_columns[:update] = timestamp_attributes_for_update_in_model
985
1019
  else
986
- instance = new
1020
+ instance = allocate
987
1021
  timestamp_columns[:create] = instance.send(:timestamp_attributes_for_create_in_model)
988
1022
  timestamp_columns[:update] = instance.send(:timestamp_attributes_for_update_in_model)
989
1023
  end
@@ -1,3 +1,5 @@
1
+ require 'active_support/core_ext/array'
2
+
1
3
  module ActiveRecord::Import
2
4
  class ValueSetTooLargeError < StandardError
3
5
  attr_reader :size
@@ -1,5 +1,5 @@
1
1
  module ActiveRecord
2
2
  module Import
3
- VERSION = "1.0.2".freeze
3
+ VERSION = "1.1.0".freeze
4
4
  end
5
5
  end
data/test/import_test.rb CHANGED
@@ -252,6 +252,16 @@ describe "#import" do
252
252
  end
253
253
  end
254
254
 
255
+ it "should index the failed instances by their poistion in the set if `track_failures` is true" do
256
+ index_offset = valid_values.length
257
+ results = Topic.import columns, valid_values + invalid_values, validate: true, track_validation_failures: true
258
+ assert_equal invalid_values.size, results.failed_instances.size
259
+ invalid_values.each_with_index do |value_set, index|
260
+ assert_equal index + index_offset, results.failed_instances[index].first
261
+ assert_equal value_set.first, results.failed_instances[index].last.title
262
+ end
263
+ end
264
+
255
265
  it "should set ids in valid models if adapter supports setting primary key of imported objects" do
256
266
  if ActiveRecord::Base.supports_setting_primary_key_of_imported_objects?
257
267
  Topic.import (invalid_models + valid_models), validate: true
@@ -395,6 +405,15 @@ describe "#import" do
395
405
  assert_equal 3, result.num_inserts if Topic.supports_import?
396
406
  end
397
407
  end
408
+
409
+ it "should accept and call an optional callable to run after each batch" do
410
+ lambda_called = 0
411
+
412
+ my_proc = ->(_row_count, _batches, _batch, _duration) { lambda_called += 1 }
413
+ Topic.import Build(10, :topics), batch_size: 4, batch_progress: my_proc
414
+
415
+ assert_equal 3, lambda_called
416
+ end
398
417
  end
399
418
 
400
419
  context "with :synchronize option" do
@@ -900,4 +919,33 @@ describe "#import" do
900
919
  end
901
920
  end
902
921
  end
922
+ describe "importing model with after_initialize callback" do
923
+ let(:columns) { %w(name size) }
924
+ let(:valid_values) { [%w("Deer", "Small"), %w("Monkey", "Medium")] }
925
+ let(:invalid_values) do
926
+ [
927
+ { name: "giraffe", size: "Large" },
928
+ { size: "Medium" } # name is missing
929
+ ]
930
+ end
931
+ context "with validation checks turned off" do
932
+ it "should import valid data" do
933
+ Animal.import(columns, valid_values, validate: false)
934
+ assert_equal 2, Animal.count
935
+ end
936
+ it "should raise ArgumentError" do
937
+ assert_raise(ArgumentError) { Animal.import(invalid_values, validate: false) }
938
+ end
939
+ end
940
+
941
+ context "with validation checks turned on" do
942
+ it "should import valid data" do
943
+ Animal.import(columns, valid_values, validate: true)
944
+ assert_equal 2, Animal.count
945
+ end
946
+ it "should raise ArgumentError" do
947
+ assert_raise(ArgumentError) { Animal.import(invalid_values, validate: true) }
948
+ end
949
+ end
950
+ end
903
951
  end
@@ -0,0 +1,6 @@
1
+ class Animal < ActiveRecord::Base
2
+ after_initialize :validate_name_presence, if: :new_record?
3
+ def validate_name_presence
4
+ raise ArgumentError if name.nil?
5
+ end
6
+ end
@@ -3,8 +3,20 @@ ActiveRecord::Schema.define do
3
3
  execute('CREATE extension IF NOT EXISTS "pgcrypto";')
4
4
  execute('CREATE extension IF NOT EXISTS "uuid-ossp";')
5
5
 
6
+ # create ENUM if it does not exist yet
7
+ begin
8
+ execute('CREATE TYPE vendor_type AS ENUM (\'wholesaler\', \'retailer\');')
9
+ rescue ActiveRecord::StatementInvalid => e
10
+ # since PostgreSQL does not support IF NOT EXISTS when creating a TYPE,
11
+ # rescue the error and check the error class
12
+ raise unless e.cause.is_a? PG::DuplicateObject
13
+ execute('ALTER TYPE vendor_type ADD VALUE IF NOT EXISTS \'wholesaler\';')
14
+ execute('ALTER TYPE vendor_type ADD VALUE IF NOT EXISTS \'retailer\';')
15
+ end
16
+
6
17
  create_table :vendors, id: :uuid, force: :cascade do |t|
7
18
  t.string :name, null: true
19
+ t.text :hours
8
20
  t.text :preferences
9
21
 
10
22
  if t.respond_to?(:json)
@@ -29,6 +41,8 @@ ActiveRecord::Schema.define do
29
41
  t.text :json_data
30
42
  end
31
43
 
44
+ t.column :vendor_type, :vendor_type
45
+
32
46
  t.datetime :created_at
33
47
  t.datetime :updated_at
34
48
  end
@@ -116,6 +116,26 @@ def should_support_postgresql_import_functionality
116
116
  assert_equal [%w(King It)], result.results
117
117
  end
118
118
 
119
+ context "when given an empty array" do
120
+ let(:result) { Book.import([], returning: %w(title)) }
121
+
122
+ setup { result }
123
+
124
+ it "returns empty arrays for ids and results" do
125
+ assert_equal [], result.ids
126
+ assert_equal [], result.results
127
+ end
128
+ end
129
+
130
+ context "when a returning column is a serialized attribute" do
131
+ let(:vendor) { Vendor.new(hours: { monday: '8-5' }) }
132
+ let(:result) { Vendor.import([vendor], returning: %w(hours)) }
133
+
134
+ it "creates records" do
135
+ assert_difference("Vendor.count", +1) { result }
136
+ end
137
+ end
138
+
119
139
  context "when primary key and returning overlap" do
120
140
  let(:result) { Book.import(books, returning: %w(id title)) }
121
141
 
@@ -249,6 +269,17 @@ def should_support_postgresql_import_functionality
249
269
  end
250
270
  end
251
271
 
272
+ describe "with enum field" do
273
+ let(:vendor_type) { "retailer" }
274
+ it "imports the correct values for enum fields" do
275
+ vendor = Vendor.new(name: 'Vendor 1', vendor_type: vendor_type)
276
+ assert_difference "Vendor.count", +1 do
277
+ Vendor.import [vendor]
278
+ end
279
+ assert_equal(vendor_type, Vendor.first.vendor_type)
280
+ end
281
+ end
282
+
252
283
  describe "with binary field" do
253
284
  let(:binary_value) { "\xE0'c\xB2\xB0\xB3Bh\\\xC2M\xB1m\\I\xC4r".force_encoding('ASCII-8BIT') }
254
285
  it "imports the correct values for binary fields" do
@@ -73,6 +73,16 @@ def should_support_basic_on_duplicate_key_update
73
73
  assert_equal user.name, users[i].name + ' Rothschild'
74
74
  assert_equal 1, user.lock_version
75
75
  end
76
+ updated_values2 = User.all.map do |user|
77
+ user.name += ' jr.'
78
+ { id: user.id, name: user.name }
79
+ end
80
+ User.import(updated_values2, on_duplicate_key_update: [:name])
81
+ assert User.count == updated_values2.length
82
+ User.all.each_with_index do |user, i|
83
+ assert_equal user.name, users[i].name + ' Rothschild jr.'
84
+ assert_equal 2, user.lock_version
85
+ end
76
86
  end
77
87
 
78
88
  it 'upsert optimistic lock columns other than lock_version by model' do
@@ -138,6 +138,15 @@ def should_support_recursive_import
138
138
  books.each do |book|
139
139
  assert_equal book.topic_id, second_new_topic.id
140
140
  end
141
+
142
+ books.each { |book| book.topic_id = nil }
143
+ assert_no_difference "Book.count", books.size do
144
+ Book.import books, validate: false, on_duplicate_key_update: [:topic_id]
145
+ end
146
+
147
+ books.each do |book|
148
+ assert_equal book.topic_id, nil
149
+ end
141
150
  end
142
151
 
143
152
  unless ENV["SKIP_COMPOSITE_PK"]
data/test/test_helper.rb CHANGED
@@ -48,7 +48,15 @@ adapter = ENV["ARE_DB"] || "sqlite3"
48
48
  FileUtils.mkdir_p 'log'
49
49
  ActiveRecord::Base.logger = Logger.new("log/test.log")
50
50
  ActiveRecord::Base.logger.level = Logger::DEBUG
51
- ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
51
+
52
+ if ENV['AR_VERSION'].to_f >= 6.0
53
+ yaml_config = YAML.load_file(test_dir.join("database.yml"))[adapter]
54
+ config = ActiveRecord::DatabaseConfigurations::HashConfig.new("test", adapter, yaml_config)
55
+ ActiveRecord::Base.configurations.configurations << config
56
+ else
57
+ ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
58
+ end
59
+
52
60
  ActiveRecord::Base.default_timezone = :utc
53
61
 
54
62
  require "activerecord-import"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: activerecord-import
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.2
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Zach Dennis
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-06-01 00:00:00.000000000 Z
11
+ date: 2021-05-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activerecord
@@ -67,7 +67,7 @@ files:
67
67
  - benchmarks/models/test_innodb.rb
68
68
  - benchmarks/models/test_memory.rb
69
69
  - benchmarks/models/test_myisam.rb
70
- - benchmarks/schema/mysql_schema.rb
70
+ - benchmarks/schema/mysql2_schema.rb
71
71
  - gemfiles/3.2.gemfile
72
72
  - gemfiles/4.0.gemfile
73
73
  - gemfiles/4.1.gemfile
@@ -121,6 +121,7 @@ files:
121
121
  - test/makara_postgis/import_test.rb
122
122
  - test/models/account.rb
123
123
  - test/models/alarm.rb
124
+ - test/models/animal.rb
124
125
  - test/models/bike_maker.rb
125
126
  - test/models/book.rb
126
127
  - test/models/car.rb
@@ -166,11 +167,11 @@ files:
166
167
  - test/travis/database.yml
167
168
  - test/value_sets_bytes_parser_test.rb
168
169
  - test/value_sets_records_parser_test.rb
169
- homepage: http://github.com/zdennis/activerecord-import
170
+ homepage: https://github.com/zdennis/activerecord-import
170
171
  licenses:
171
- - Ruby
172
+ - MIT
172
173
  metadata: {}
173
- post_install_message:
174
+ post_install_message:
174
175
  rdoc_options: []
175
176
  require_paths:
176
177
  - lib
@@ -178,16 +179,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
178
179
  requirements:
179
180
  - - ">="
180
181
  - !ruby/object:Gem::Version
181
- version: 1.9.2
182
+ version: 2.0.0
182
183
  required_rubygems_version: !ruby/object:Gem::Requirement
183
184
  requirements:
184
185
  - - ">="
185
186
  - !ruby/object:Gem::Version
186
187
  version: '0'
187
188
  requirements: []
188
- rubyforge_project:
189
- rubygems_version: 2.7.8
190
- signing_key:
189
+ rubygems_version: 3.0.8
190
+ signing_key:
191
191
  specification_version: 4
192
192
  summary: Bulk insert extension for ActiveRecord
193
193
  test_files:
@@ -212,6 +212,7 @@ test_files:
212
212
  - test/makara_postgis/import_test.rb
213
213
  - test/models/account.rb
214
214
  - test/models/alarm.rb
215
+ - test/models/animal.rb
215
216
  - test/models/bike_maker.rb
216
217
  - test/models/book.rb
217
218
  - test/models/car.rb