activerecord-import 1.0.2 → 1.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yaml +151 -0
- data/.gitignore +5 -0
- data/.rubocop.yml +74 -8
- data/.rubocop_todo.yml +10 -16
- data/Brewfile +3 -1
- data/CHANGELOG.md +138 -3
- data/Dockerfile +23 -0
- data/Gemfile +24 -14
- data/LICENSE +21 -56
- data/README.markdown +108 -60
- data/Rakefile +3 -0
- data/activerecord-import.gemspec +6 -5
- data/benchmarks/benchmark.rb +10 -4
- data/benchmarks/lib/base.rb +4 -2
- data/benchmarks/lib/cli_parser.rb +4 -2
- data/benchmarks/lib/float.rb +2 -0
- data/benchmarks/lib/mysql2_benchmark.rb +2 -0
- data/benchmarks/lib/output_to_csv.rb +2 -0
- data/benchmarks/lib/output_to_html.rb +4 -2
- data/benchmarks/models/test_innodb.rb +2 -0
- data/benchmarks/models/test_memory.rb +2 -0
- data/benchmarks/models/test_myisam.rb +2 -0
- data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
- data/docker-compose.yml +34 -0
- data/gemfiles/4.2.gemfile +2 -0
- data/gemfiles/5.0.gemfile +2 -0
- data/gemfiles/5.1.gemfile +2 -0
- data/gemfiles/5.2.gemfile +2 -0
- data/gemfiles/6.0.gemfile +4 -1
- data/gemfiles/6.1.gemfile +4 -1
- data/gemfiles/7.0.gemfile +4 -0
- data/gemfiles/7.1.gemfile +3 -0
- data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
- data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/trilogy_adapter.rb +8 -0
- data/lib/activerecord-import/adapters/abstract_adapter.rb +14 -5
- data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +33 -25
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +69 -56
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +39 -39
- data/lib/activerecord-import/adapters/trilogy_adapter.rb +7 -0
- data/lib/activerecord-import/base.rb +10 -2
- data/lib/activerecord-import/import.rb +162 -65
- data/lib/activerecord-import/mysql2.rb +2 -0
- data/lib/activerecord-import/postgresql.rb +2 -0
- data/lib/activerecord-import/sqlite3.rb +2 -0
- data/lib/activerecord-import/synchronize.rb +3 -1
- data/lib/activerecord-import/value_sets_parser.rb +5 -0
- data/lib/activerecord-import/version.rb +3 -1
- data/lib/activerecord-import.rb +2 -1
- data/test/adapters/jdbcmysql.rb +2 -0
- data/test/adapters/jdbcpostgresql.rb +2 -0
- data/test/adapters/jdbcsqlite3.rb +2 -0
- data/test/adapters/makara_postgis.rb +2 -0
- data/test/adapters/mysql2.rb +2 -0
- data/test/adapters/mysql2_makara.rb +2 -0
- data/test/adapters/mysql2spatial.rb +2 -0
- data/test/adapters/postgis.rb +2 -0
- data/test/adapters/postgresql.rb +2 -0
- data/test/adapters/postgresql_makara.rb +2 -0
- data/test/adapters/seamless_database_pool.rb +2 -0
- data/test/adapters/spatialite.rb +2 -0
- data/test/adapters/sqlite3.rb +2 -0
- data/test/adapters/trilogy.rb +9 -0
- data/test/database.yml.sample +7 -0
- data/test/{travis → github}/database.yml +7 -1
- data/test/import_test.rb +93 -2
- data/test/jdbcmysql/import_test.rb +5 -3
- data/test/jdbcpostgresql/import_test.rb +4 -2
- data/test/jdbcsqlite3/import_test.rb +4 -2
- data/test/makara_postgis/import_test.rb +4 -2
- data/test/models/account.rb +2 -0
- data/test/models/alarm.rb +2 -0
- data/test/models/animal.rb +8 -0
- data/test/models/author.rb +7 -0
- data/test/models/bike_maker.rb +3 -0
- data/test/models/book.rb +7 -2
- data/test/models/car.rb +2 -0
- data/test/models/card.rb +5 -0
- data/test/models/chapter.rb +2 -0
- data/test/models/composite_book.rb +19 -0
- data/test/models/composite_chapter.rb +9 -0
- data/test/models/customer.rb +18 -0
- data/test/models/deck.rb +8 -0
- data/test/models/dictionary.rb +2 -0
- data/test/models/discount.rb +2 -0
- data/test/models/end_note.rb +2 -0
- data/test/models/group.rb +2 -0
- data/test/models/order.rb +17 -0
- data/test/models/playing_card.rb +4 -0
- data/test/models/promotion.rb +2 -0
- data/test/models/question.rb +2 -0
- data/test/models/rule.rb +2 -0
- data/test/models/tag.rb +9 -1
- data/test/models/tag_alias.rb +11 -0
- data/test/models/topic.rb +7 -0
- data/test/models/user.rb +2 -0
- data/test/models/user_token.rb +2 -0
- data/test/models/vendor.rb +2 -0
- data/test/models/widget.rb +2 -0
- data/test/mysql2/import_test.rb +5 -3
- data/test/mysql2_makara/import_test.rb +5 -3
- data/test/mysqlspatial2/import_test.rb +5 -3
- data/test/postgis/import_test.rb +4 -2
- data/test/postgresql/import_test.rb +4 -2
- data/test/schema/generic_schema.rb +37 -1
- data/test/schema/jdbcpostgresql_schema.rb +3 -1
- data/test/schema/mysql2_schema.rb +2 -0
- data/test/schema/postgis_schema.rb +3 -1
- data/test/schema/postgresql_schema.rb +47 -0
- data/test/schema/sqlite3_schema.rb +2 -0
- data/test/schema/version.rb +2 -0
- data/test/sqlite3/import_test.rb +4 -2
- data/test/support/active_support/test_case_extensions.rb +2 -0
- data/test/support/assertions.rb +2 -0
- data/test/support/factories.rb +2 -0
- data/test/support/generate.rb +4 -2
- data/test/support/mysql/import_examples.rb +2 -1
- data/test/support/postgresql/import_examples.rb +108 -2
- data/test/support/shared_examples/on_duplicate_key_ignore.rb +2 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +78 -9
- data/test/support/shared_examples/recursive_import.rb +98 -1
- data/test/support/sqlite3/import_examples.rb +2 -1
- data/test/synchronize_test.rb +2 -0
- data/test/test_helper.rb +33 -6
- data/test/trilogy/import_test.rb +7 -0
- data/test/value_sets_bytes_parser_test.rb +3 -1
- data/test/value_sets_records_parser_test.rb +3 -1
- metadata +42 -16
- data/.travis.yml +0 -70
- data/gemfiles/3.2.gemfile +0 -2
- data/gemfiles/4.0.gemfile +0 -2
- data/gemfiles/4.1.gemfile +0 -2
data/LICENSE
CHANGED
@@ -1,56 +1,21 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
d) make other distribution arrangements with the author.
|
24
|
-
|
25
|
-
3. You may distribute the software in object code or binary form,
|
26
|
-
provided that you do at least ONE of the following:
|
27
|
-
|
28
|
-
a) distribute the binaries and library files of the software,
|
29
|
-
together with instructions (in the manual page or equivalent)
|
30
|
-
on where to get the original distribution.
|
31
|
-
|
32
|
-
b) accompany the distribution with the machine-readable source of
|
33
|
-
the software.
|
34
|
-
|
35
|
-
c) give non-standard binaries non-standard names, with
|
36
|
-
instructions on where to get the original software distribution.
|
37
|
-
|
38
|
-
d) make other distribution arrangements with the author.
|
39
|
-
|
40
|
-
4. You may modify and include the part of the software into any other
|
41
|
-
software (possibly commercial). But some files in the distribution
|
42
|
-
are not written by the author, so that they are not under these terms.
|
43
|
-
|
44
|
-
For the list of those files and their copying conditions, see the
|
45
|
-
file LEGAL.
|
46
|
-
|
47
|
-
5. The scripts and library files supplied as input to or produced as
|
48
|
-
output from the software do not automatically fall under the
|
49
|
-
copyright of the software, but belong to whomever generated them,
|
50
|
-
and may be sold commercially, and may be aggregated with this
|
51
|
-
software.
|
52
|
-
|
53
|
-
6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
|
54
|
-
IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
|
55
|
-
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
56
|
-
PURPOSE.
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2020 Zach Dennis <zach.dennis@gmail.com>
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.markdown
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
#
|
1
|
+
# Activerecord-Import ![Build Status](https://github.com/zdennis/activerecord-import/actions/workflows/test.yaml/badge.svg)
|
2
2
|
|
3
|
-
|
3
|
+
Activerecord-Import is a library for bulk inserting data using ActiveRecord.
|
4
4
|
|
5
5
|
One of its major features is following activerecord associations and generating the minimal
|
6
6
|
number of SQL insert statements required, avoiding the N+1 insert problem. An example probably
|
@@ -23,10 +23,10 @@ an 18 hour batch process to <2 hrs.
|
|
23
23
|
|
24
24
|
The gem provides the following high-level features:
|
25
25
|
|
26
|
-
*
|
27
|
-
*
|
28
|
-
*
|
29
|
-
*
|
26
|
+
* Works with raw columns and arrays of values (fastest)
|
27
|
+
* Works with model objects (faster)
|
28
|
+
* Performs validations (fast)
|
29
|
+
* Performs on duplicate key updates (requires MySQL, SQLite 3.24.0+, or Postgres 9.5+)
|
30
30
|
|
31
31
|
## Table of Contents
|
32
32
|
|
@@ -54,11 +54,14 @@ The gem provides the following high-level features:
|
|
54
54
|
* [More Information](#more-information)
|
55
55
|
* [Contributing](#contributing)
|
56
56
|
* [Running Tests](#running-tests)
|
57
|
+
* [Issue Triage](#issue-triage)
|
57
58
|
|
58
59
|
### Examples
|
59
60
|
|
60
61
|
#### Introduction
|
61
62
|
|
63
|
+
This gem adds an `import` method (or `bulk_import`, for compatibility with gems like `elasticsearch-model`; see [Conflicts With Other Gems](#conflicts-with-other-gems)) to ActiveRecord classes.
|
64
|
+
|
62
65
|
Without `activerecord-import`, you'd write something like this:
|
63
66
|
|
64
67
|
```ruby
|
@@ -85,7 +88,7 @@ The `import` method can take an array of column names (string or symbols) and an
|
|
85
88
|
|
86
89
|
```ruby
|
87
90
|
columns = [ :title, :author ]
|
88
|
-
values = [ ['Book1', '
|
91
|
+
values = [ ['Book1', 'George Orwell'], ['Book2', 'Bob Jones'] ]
|
89
92
|
|
90
93
|
# Importing without model validations
|
91
94
|
Book.import columns, values, validate: false
|
@@ -102,7 +105,7 @@ Book.import columns, values
|
|
102
105
|
The `import` method can take an array of hashes. The keys map to the column names in the database.
|
103
106
|
|
104
107
|
```ruby
|
105
|
-
values = [{ title: 'Book1', author: '
|
108
|
+
values = [{ title: 'Book1', author: 'George Orwell' }, { title: 'Book2', author: 'Bob Jones'}]
|
106
109
|
|
107
110
|
# Importing without model validations
|
108
111
|
Book.import values, validate: false
|
@@ -119,7 +122,7 @@ The `import` method can take an array of column names and an array of hash objec
|
|
119
122
|
|
120
123
|
```ruby
|
121
124
|
books = [
|
122
|
-
{ title: "Book 1", author: "
|
125
|
+
{ title: "Book 1", author: "George Orwell" },
|
123
126
|
{ title: "Book 2", author: "Bob Jones" }
|
124
127
|
]
|
125
128
|
columns = [ :title ]
|
@@ -171,7 +174,7 @@ The `import` method can take an array of models. The attributes will be pulled o
|
|
171
174
|
|
172
175
|
```ruby
|
173
176
|
books = [
|
174
|
-
Book.new(title: "Book 1", author: "
|
177
|
+
Book.new(title: "Book 1", author: "George Orwell"),
|
175
178
|
Book.new(title: "Book 2", author: "Bob Jones")
|
176
179
|
]
|
177
180
|
|
@@ -189,7 +192,7 @@ The `import` method can take an array of column names and an array of models. Th
|
|
189
192
|
|
190
193
|
```ruby
|
191
194
|
books = [
|
192
|
-
Book.new(title: "Book 1", author: "
|
195
|
+
Book.new(title: "Book 1", author: "George Orwell"),
|
193
196
|
Book.new(title: "Book 2", author: "Bob Jones")
|
194
197
|
]
|
195
198
|
columns = [ :title ]
|
@@ -217,7 +220,7 @@ The `import` method can take a `batch_size` option to control the number of rows
|
|
217
220
|
|
218
221
|
```ruby
|
219
222
|
books = [
|
220
|
-
Book.new(title: "Book 1", author: "
|
223
|
+
Book.new(title: "Book 1", author: "George Orwell"),
|
221
224
|
Book.new(title: "Book 2", author: "Bob Jones"),
|
222
225
|
Book.new(title: "Book 1", author: "John Doe"),
|
223
226
|
Book.new(title: "Book 2", author: "Richard Wright")
|
@@ -228,9 +231,22 @@ columns = [ :title ]
|
|
228
231
|
Book.import columns, books, batch_size: 2
|
229
232
|
```
|
230
233
|
|
234
|
+
If your import is particularly large or slow (possibly due to [callbacks](#callbacks)) whilst batch importing, you might want a way to report back on progress. This is supported by passing a callable as the `batch_progress` option. e.g:
|
235
|
+
|
236
|
+
```ruby
|
237
|
+
my_proc = ->(rows_size, num_batches, current_batch_number, batch_duration_in_secs) {
|
238
|
+
# Using the arguments provided to the callable, you can
|
239
|
+
# send an email, post to a websocket,
|
240
|
+
# update slack, alert if import is taking too long, etc.
|
241
|
+
}
|
242
|
+
|
243
|
+
Book.import columns, books, batch_size: 2, batch_progress: my_proc
|
244
|
+
```
|
245
|
+
|
231
246
|
#### Recursive
|
232
247
|
|
233
|
-
|
248
|
+
> **Note**
|
249
|
+
> This only works with PostgreSQL and ActiveRecord objects. This won't work with hashes or arrays as recursive inputs.
|
234
250
|
|
235
251
|
Assume that Books <code>has_many</code> Reviews.
|
236
252
|
|
@@ -246,19 +262,22 @@ Book.import books, recursive: true
|
|
246
262
|
|
247
263
|
### Options
|
248
264
|
|
249
|
-
Key
|
250
|
-
|
251
|
-
:validate
|
252
|
-
:validate_uniqueness
|
253
|
-
:
|
254
|
-
:
|
255
|
-
:
|
256
|
-
:
|
257
|
-
:
|
258
|
-
:
|
259
|
-
:
|
260
|
-
:
|
261
|
-
|
265
|
+
Key | Options | Default | Description
|
266
|
+
------------------------- | --------------------- | ------------------ | -----------
|
267
|
+
:validate | `true`/`false` | `true` | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
|
268
|
+
:validate_uniqueness | `true`/`false` | `false` | Whether or not to run ActiveRecord uniqueness validations. Beware this will incur an sql query per-record (N+1 queries). (requires `>= v0.27.0`).
|
269
|
+
:validate_with_context | `Symbol` |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
|
270
|
+
:track_validation_failures| `true`/`false` | `false` | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]`
|
271
|
+
:on_duplicate_key_ignore | `true`/`false` | `false` | Allows skipping records with duplicate keys. See [here](#duplicate-key-ignore) for more details.
|
272
|
+
:ignore | `true`/`false` | `false` | Alias for :on_duplicate_key_ignore.
|
273
|
+
:on_duplicate_key_update | :all, `Array`, `Hash` | N/A | Allows upsert logic to be used. See [here](#duplicate-key-update) for more details.
|
274
|
+
:synchronize | `Array` | N/A | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
|
275
|
+
:timestamps | `true`/`false` | `true` | Enables/disables timestamps on imported records.
|
276
|
+
:recursive | `true`/`false` | `false` | Imports has_many/has_one associations (PostgreSQL only).
|
277
|
+
:recursive_on_duplicate_key_update | `Hash` | N/A | Allows upsert logic to be used for recursive associations. The hash key is the association name and the value has the same options as `:on_duplicate_key_update`. See [here](#duplicate-key-update) for more details.
|
278
|
+
:batch_size | `Integer` | total # of records | Max number of records to insert per import
|
279
|
+
:raise_error | `true`/`false` | `false` | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
|
280
|
+
:all_or_none | `true`/`false` | `false` | Will not import any records if there is a record with validation errors.
|
262
281
|
|
263
282
|
#### Duplicate Key Ignore
|
264
283
|
|
@@ -267,14 +286,14 @@ Key | Options | Default | Descripti
|
|
267
286
|
For Postgres 9.5+ it adds `ON CONFLICT DO NOTHING`, for MySQL it uses `INSERT IGNORE`, and for SQLite it uses `INSERT OR IGNORE`. Cannot be enabled on a recursive import. For database adapters that normally support setting primary keys on imported objects, this option prevents that from occurring.
|
268
287
|
|
269
288
|
```ruby
|
270
|
-
book = Book.create! title: "Book1", author: "
|
289
|
+
book = Book.create! title: "Book1", author: "George Orwell"
|
271
290
|
book.title = "Updated Book Title"
|
272
291
|
book.author = "Bob Barker"
|
273
292
|
|
274
293
|
Book.import [book], on_duplicate_key_ignore: true
|
275
294
|
|
276
295
|
book.reload.title # => "Book1" (stayed the same)
|
277
|
-
book.reload.author # => "
|
296
|
+
book.reload.author # => "George Orwell" (stayed the same)
|
278
297
|
```
|
279
298
|
|
280
299
|
The option `:on_duplicate_key_ignore` is bypassed when `:recursive` is enabled for [PostgreSQL imports](https://github.com/zdennis/activerecord-import/wiki#recursive-example-postgresql-only).
|
@@ -290,7 +309,7 @@ This will use MySQL's `ON DUPLICATE KEY UPDATE` or Postgres/SQLite `ON CONFLICT
|
|
290
309
|
Basic Update
|
291
310
|
|
292
311
|
```ruby
|
293
|
-
book = Book.create! title: "Book1", author: "
|
312
|
+
book = Book.create! title: "Book1", author: "George Orwell"
|
294
313
|
book.title = "Updated Book Title"
|
295
314
|
book.author = "Bob Barker"
|
296
315
|
|
@@ -304,13 +323,13 @@ Book.import [book], on_duplicate_key_update: {conflict_target: [:id], columns: [
|
|
304
323
|
Book.import [book], on_duplicate_key_update: [:title]
|
305
324
|
|
306
325
|
book.reload.title # => "Updated Book Title" (changed)
|
307
|
-
book.reload.author # => "
|
326
|
+
book.reload.author # => "George Orwell" (stayed the same)
|
308
327
|
```
|
309
328
|
|
310
329
|
Using the value from another column
|
311
330
|
|
312
331
|
```ruby
|
313
|
-
book = Book.create! title: "Book1", author: "
|
332
|
+
book = Book.create! title: "Book1", author: "George Orwell"
|
314
333
|
book.title = "Updated Book Title"
|
315
334
|
|
316
335
|
# MySQL version
|
@@ -328,7 +347,7 @@ book.reload.author # => "Updated Book Title" (changed)
|
|
328
347
|
Using Custom SQL
|
329
348
|
|
330
349
|
```ruby
|
331
|
-
book = Book.create! title: "Book1", author: "
|
350
|
+
book = Book.create! title: "Book1", author: "George Orwell"
|
332
351
|
book.author = "Bob Barker"
|
333
352
|
|
334
353
|
# MySQL version
|
@@ -346,10 +365,33 @@ book.reload.title # => "Book1" (stayed the same)
|
|
346
365
|
book.reload.author # => "Bob Barker" (changed)
|
347
366
|
```
|
348
367
|
|
368
|
+
PostgreSQL Using partial indexes
|
369
|
+
|
370
|
+
```ruby
|
371
|
+
book = Book.create! title: "Book1", author: "George Orwell", published_at: Time.now
|
372
|
+
book.author = "Bob Barker"
|
373
|
+
|
374
|
+
# in migration
|
375
|
+
execute <<-SQL
|
376
|
+
CREATE INDEX books_published_at_index ON books (published_at) WHERE published_at IS NOT NULL;
|
377
|
+
SQL
|
378
|
+
|
379
|
+
# PostgreSQL version
|
380
|
+
Book.import [book], on_duplicate_key_update: {
|
381
|
+
conflict_target: [:id],
|
382
|
+
index_predicate: "published_at IS NOT NULL",
|
383
|
+
columns: [:author]
|
384
|
+
}
|
385
|
+
|
386
|
+
book.reload.title # => "Book1" (stayed the same)
|
387
|
+
book.reload.author # => "Bob Barker" (changed)
|
388
|
+
book.reload.published_at # => 2017-10-09 (stayed the same)
|
389
|
+
```
|
390
|
+
|
349
391
|
PostgreSQL Using constraints
|
350
392
|
|
351
393
|
```ruby
|
352
|
-
book = Book.create! title: "Book1", author: "
|
394
|
+
book = Book.create! title: "Book1", author: "George Orwell", edition: 3, published_at: nil
|
353
395
|
book.published_at = Time.now
|
354
396
|
|
355
397
|
# in migration
|
@@ -363,7 +405,7 @@ Book.import [book], on_duplicate_key_update: {constraint_name: :for_upsert, colu
|
|
363
405
|
|
364
406
|
|
365
407
|
book.reload.title # => "Book1" (stayed the same)
|
366
|
-
book.reload.author # => "
|
408
|
+
book.reload.author # => "George Orwell" (stayed the same)
|
367
409
|
book.reload.edition # => 3 (stayed the same)
|
368
410
|
book.reload.published_at # => 2017-10-09 (changed)
|
369
411
|
```
|
@@ -374,7 +416,7 @@ Book.import books, validate_uniqueness: true
|
|
374
416
|
|
375
417
|
### Return Info
|
376
418
|
|
377
|
-
The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed
|
419
|
+
The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed.
|
378
420
|
|
379
421
|
```ruby
|
380
422
|
articles = [
|
@@ -415,7 +457,9 @@ Should you wish to specify those columns, you may use the option `timestamps: fa
|
|
415
457
|
|
416
458
|
However, it is also possible to set just `:created_at` in specific records. In this case despite using `timestamps: true`, `:created_at` will be updated only in records where that field is `nil`. Same rule applies for record associations when enabling the option `recursive: true`.
|
417
459
|
|
418
|
-
If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is
|
460
|
+
If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is.
|
461
|
+
> **Note**
|
462
|
+
> If you are using ActiveRecord 7.0 or later, please use `ActiveRecord.default_timezone` instead.
|
419
463
|
|
420
464
|
### Callbacks
|
421
465
|
|
@@ -487,7 +531,8 @@ This allows an external gem to dynamically add an adapter without the need to ad
|
|
487
531
|
|
488
532
|
### Requiring
|
489
533
|
|
490
|
-
Note
|
534
|
+
> **Note**
|
535
|
+
> These instructions will only work if you are using version 0.2.0 or higher.
|
491
536
|
|
492
537
|
#### Autoloading via Bundler
|
493
538
|
|
@@ -524,7 +569,7 @@ require 'activerecord-import'
|
|
524
569
|
### Load Path Setup
|
525
570
|
To understand how rubygems loads code you can reference the following:
|
526
571
|
|
527
|
-
http://guides.rubygems.org/patterns/#
|
572
|
+
http://guides.rubygems.org/patterns/#loading-code
|
528
573
|
|
529
574
|
And an example of how active_record dynamically load adapters:
|
530
575
|
|
@@ -552,9 +597,9 @@ When rubygems pushes the `lib` folder onto the load path a `require` will now fi
|
|
552
597
|
|
553
598
|
### Conflicts With Other Gems
|
554
599
|
|
555
|
-
|
600
|
+
Activerecord-Import adds the `.import` method onto `ActiveRecord::Base`. There are other gems, such as `elasticsearch-rails`, that do the same thing. In conflicts such as this, there is an aliased method named `.bulk_import` that can be used interchangeably.
|
556
601
|
|
557
|
-
If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by
|
602
|
+
If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by explicitly setting this value within the model. For example:
|
558
603
|
|
559
604
|
```ruby
|
560
605
|
class Post < ActiveRecord::Base
|
@@ -579,7 +624,7 @@ See https://github.com/zdennis/activerecord-import/issues/233 for further discus
|
|
579
624
|
|
580
625
|
### More Information
|
581
626
|
|
582
|
-
For more information on
|
627
|
+
For more information on Activerecord-Import please see its wiki: https://github.com/zdennis/activerecord-import/wiki
|
583
628
|
|
584
629
|
To document new information, please add to the README instead of the wiki. See https://github.com/zdennis/activerecord-import/issues/397 for discussion.
|
585
630
|
|
@@ -597,32 +642,35 @@ After that, you can run the tests. They run against multiple tests and ActiveRec
|
|
597
642
|
|
598
643
|
This is one example of how to run the tests:
|
599
644
|
|
600
|
-
```
|
645
|
+
```bash
|
601
646
|
rm Gemfile.lock
|
602
|
-
AR_VERSION=
|
603
|
-
AR_VERSION=
|
647
|
+
AR_VERSION=7.0 bundle install
|
648
|
+
AR_VERSION=7.0 bundle exec rake test:postgresql test:sqlite3 test:mysql2
|
604
649
|
```
|
605
650
|
|
606
|
-
Once you have pushed up your changes, you can find your CI results [here](https://
|
651
|
+
Once you have pushed up your changes, you can find your CI results [here](https://github.com/zdennis/activerecord-import/actions).
|
652
|
+
|
653
|
+
#### Docker Setup
|
654
|
+
|
655
|
+
Before you begin, make sure you have [Docker](https://www.docker.com/products/docker-desktop/) and [Docker Compose](https://docs.docker.com/compose/) installed on your machine. If you don't, you can install both via Homebrew using the following command:
|
656
|
+
|
657
|
+
```bash
|
658
|
+
brew install docker && brew install docker-compose
|
659
|
+
```
|
660
|
+
##### Steps
|
661
|
+
|
662
|
+
1. In your terminal run `docker-compose up --build`
|
663
|
+
1. In another tab/window run `docker-compose exec app bash`
|
664
|
+
1. In that same terminal run the mysql2 test by running `bundle exec rake test:mysql2`
|
665
|
+
|
666
|
+
## Issue Triage [![Open Source Helpers](https://www.codetriage.com/zdennis/activerecord-import/badges/users.svg)](https://www.codetriage.com/zdennis/activerecord-import)
|
667
|
+
|
668
|
+
You can triage issues which may include reproducing bug reports or asking for vital information, such as version numbers or reproduction instructions. If you would like to start triaging issues, one easy way to get started is to [subscribe to activerecord-import on CodeTriage](https://www.codetriage.com/zdennis/activerecord-import).
|
607
669
|
|
608
670
|
# License
|
609
671
|
|
610
|
-
This is licensed under the
|
672
|
+
This is licensed under the MIT license.
|
611
673
|
|
612
674
|
# Author
|
613
675
|
|
614
676
|
Zach Dennis (zach.dennis@gmail.com)
|
615
|
-
|
616
|
-
# Contributors
|
617
|
-
|
618
|
-
* Jordan Owens (@jkowens)
|
619
|
-
* Erik Michaels-Ober (@sferik)
|
620
|
-
* Blythe Dunham
|
621
|
-
* Gabe da Silveira
|
622
|
-
* Henry Work
|
623
|
-
* James Herdman
|
624
|
-
* Marcus Crafter
|
625
|
-
* Thibaud Guillaume-Gentil
|
626
|
-
* Mark Van Holstyn
|
627
|
-
* Victor Costan
|
628
|
-
* Dillon Welch
|
data/Rakefile
CHANGED
data/activerecord-import.gemspec
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require File.expand_path('../lib/activerecord-import/version', __FILE__)
|
3
4
|
|
4
5
|
Gem::Specification.new do |gem|
|
@@ -6,8 +7,8 @@ Gem::Specification.new do |gem|
|
|
6
7
|
gem.email = ["zach.dennis@gmail.com"]
|
7
8
|
gem.summary = "Bulk insert extension for ActiveRecord"
|
8
9
|
gem.description = "A library for bulk inserting data using ActiveRecord."
|
9
|
-
gem.homepage = "
|
10
|
-
gem.license = "
|
10
|
+
gem.homepage = "https://github.com/zdennis/activerecord-import"
|
11
|
+
gem.license = "MIT"
|
11
12
|
|
12
13
|
gem.files = `git ls-files`.split($\)
|
13
14
|
gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
|
@@ -16,8 +17,8 @@ Gem::Specification.new do |gem|
|
|
16
17
|
gem.require_paths = ["lib"]
|
17
18
|
gem.version = ActiveRecord::Import::VERSION
|
18
19
|
|
19
|
-
gem.required_ruby_version = ">=
|
20
|
+
gem.required_ruby_version = ">= 2.4.0"
|
20
21
|
|
21
|
-
gem.add_runtime_dependency "activerecord", ">=
|
22
|
+
gem.add_runtime_dependency "activerecord", ">= 4.2"
|
22
23
|
gem.add_development_dependency "rake"
|
23
24
|
end
|
data/benchmarks/benchmark.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'pathname'
|
2
4
|
require "fileutils"
|
3
5
|
require "active_record"
|
@@ -20,7 +22,11 @@ FileUtils.mkdir_p 'log'
|
|
20
22
|
ActiveRecord::Base.configurations["test"] = YAML.load_file(File.join(benchmark_dir, "../test/database.yml"))[options.adapter]
|
21
23
|
ActiveRecord::Base.logger = Logger.new("log/test.log")
|
22
24
|
ActiveRecord::Base.logger.level = Logger::DEBUG
|
23
|
-
ActiveRecord
|
25
|
+
if ActiveRecord.respond_to?(:default_timezone)
|
26
|
+
ActiveRecord.default_timezone = :utc
|
27
|
+
else
|
28
|
+
ActiveRecord::Base.default_timezone = :utc
|
29
|
+
end
|
24
30
|
|
25
31
|
require "activerecord-import"
|
26
32
|
ActiveRecord::Base.establish_connection(:test)
|
@@ -35,7 +41,7 @@ require File.join(benchmark_dir, "../test/schema/generic_schema")
|
|
35
41
|
adapter_schema = File.join(benchmark_dir, "schema/#{options.adapter}_schema.rb")
|
36
42
|
require adapter_schema if File.exist?(adapter_schema)
|
37
43
|
|
38
|
-
Dir[File.dirname(__FILE__)
|
44
|
+
Dir["#{File.dirname(__FILE__)}/models/*.rb"].sort.each { |file| require file }
|
39
45
|
|
40
46
|
require File.join( benchmark_dir, 'lib', "#{options.adapter}_benchmark" )
|
41
47
|
|
@@ -47,8 +53,8 @@ else
|
|
47
53
|
end
|
48
54
|
|
49
55
|
letter = options.adapter[0].chr
|
50
|
-
clazz_str = letter.upcase + options.adapter[1
|
51
|
-
clazz = Object.const_get( clazz_str
|
56
|
+
clazz_str = letter.upcase + options.adapter[1..].downcase
|
57
|
+
clazz = Object.const_get( "#{clazz_str}Benchmark" )
|
52
58
|
|
53
59
|
benchmarks = []
|
54
60
|
options.number_of_objects.each do |num|
|
data/benchmarks/lib/base.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
class BenchmarkBase
|
2
4
|
attr_reader :results
|
3
5
|
|
@@ -25,11 +27,11 @@ class BenchmarkBase
|
|
25
27
|
# An OpenStruct object with the following attributes:
|
26
28
|
# * description - the description of the benchmark ran
|
27
29
|
# * tms - a Benchmark::Tms containing the results of the benchmark
|
28
|
-
def bm( description )
|
30
|
+
def bm( description, &block )
|
29
31
|
tms = nil
|
30
32
|
puts "Benchmarking #{description}"
|
31
33
|
|
32
|
-
Benchmark.bm { |x| tms = x.report
|
34
|
+
Benchmark.bm { |x| tms = x.report(&block) }
|
33
35
|
delete_all
|
34
36
|
failed = false
|
35
37
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'optparse'
|
2
4
|
require 'ostruct'
|
3
5
|
|
@@ -8,7 +10,7 @@ require 'ostruct'
|
|
8
10
|
# * t - the table types to test. ie: myisam, innodb, memory, temporary, etc.
|
9
11
|
#
|
10
12
|
module BenchmarkOptionParser
|
11
|
-
BANNER = "Usage: ruby #{$0} [options]\nSee ruby #{$0} -h for more options."
|
13
|
+
BANNER = "Usage: ruby #{$0} [options]\nSee ruby #{$0} -h for more options."
|
12
14
|
|
13
15
|
def self.print_banner
|
14
16
|
puts BANNER
|
@@ -27,7 +29,7 @@ module BenchmarkOptionParser
|
|
27
29
|
print_valid_table_types( options, prefix: " " )
|
28
30
|
end
|
29
31
|
|
30
|
-
# TODO IMPLEMENT THIS
|
32
|
+
# TODO: IMPLEMENT THIS
|
31
33
|
def self.print_valid_table_types( options, hsh = { prefix: '' } )
|
32
34
|
if !options.table_types.keys.empty?
|
33
35
|
options.table_types.keys.sort.each { |type| puts hsh[:prefix].to_s + type.to_s }
|
data/benchmarks/lib/float.rb
CHANGED
@@ -1,7 +1,9 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'erb'
|
2
4
|
|
3
5
|
module OutputToHTML
|
4
|
-
TEMPLATE_HEADER = <<"EOT"
|
6
|
+
TEMPLATE_HEADER = <<"EOT"
|
5
7
|
<div>
|
6
8
|
All times are rounded to the nearest thousandth for display purposes. Speedups next to each time are computed
|
7
9
|
before any rounding occurs. Also, all speedup calculations are computed by comparing a given time against
|
@@ -9,7 +11,7 @@ module OutputToHTML
|
|
9
11
|
</div>
|
10
12
|
EOT
|
11
13
|
|
12
|
-
TEMPLATE = <<"EOT"
|
14
|
+
TEMPLATE = <<"EOT"
|
13
15
|
<style>
|
14
16
|
td#benchmarkTitle {
|
15
17
|
border: 1px solid black;
|