activerecord-import 1.0.4 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yaml +159 -0
- data/.gitignore +5 -0
- data/.rubocop.yml +76 -7
- data/.rubocop_todo.yml +10 -16
- data/Brewfile +3 -1
- data/CHANGELOG.md +143 -3
- data/Dockerfile +23 -0
- data/Gemfile +28 -24
- data/LICENSE +21 -56
- data/README.markdown +83 -27
- data/Rakefile +3 -0
- data/activerecord-import.gemspec +10 -5
- data/benchmarks/benchmark.rb +10 -6
- data/benchmarks/lib/base.rb +10 -5
- data/benchmarks/lib/cli_parser.rb +10 -6
- data/benchmarks/lib/float.rb +2 -0
- data/benchmarks/lib/mysql2_benchmark.rb +2 -0
- data/benchmarks/lib/output_to_csv.rb +2 -0
- data/benchmarks/lib/output_to_html.rb +4 -2
- data/benchmarks/models/test_innodb.rb +2 -0
- data/benchmarks/models/test_memory.rb +2 -0
- data/benchmarks/models/test_myisam.rb +2 -0
- data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
- data/docker-compose.yml +34 -0
- data/gemfiles/5.2.gemfile +2 -0
- data/gemfiles/6.0.gemfile +3 -0
- data/gemfiles/6.1.gemfile +4 -1
- data/gemfiles/7.0.gemfile +4 -0
- data/gemfiles/7.1.gemfile +3 -0
- data/gemfiles/7.2.gemfile +3 -0
- data/gemfiles/8.0.gemfile +3 -0
- data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
- data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/trilogy_adapter.rb +8 -0
- data/lib/activerecord-import/adapters/abstract_adapter.rb +9 -6
- data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +30 -21
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +68 -48
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +37 -30
- data/lib/activerecord-import/adapters/trilogy_adapter.rb +7 -0
- data/lib/activerecord-import/base.rb +3 -1
- data/lib/activerecord-import/import.rb +160 -58
- data/lib/activerecord-import/synchronize.rb +3 -1
- data/lib/activerecord-import/value_sets_parser.rb +5 -0
- data/lib/activerecord-import/version.rb +3 -1
- data/lib/activerecord-import.rb +2 -1
- data/test/adapters/jdbcmysql.rb +2 -0
- data/test/adapters/jdbcpostgresql.rb +2 -0
- data/test/adapters/jdbcsqlite3.rb +2 -0
- data/test/adapters/makara_postgis.rb +2 -0
- data/test/adapters/mysql2.rb +2 -0
- data/test/adapters/mysql2_makara.rb +2 -0
- data/test/adapters/mysql2spatial.rb +2 -0
- data/test/adapters/postgis.rb +2 -0
- data/test/adapters/postgresql.rb +2 -0
- data/test/adapters/postgresql_makara.rb +2 -0
- data/test/adapters/seamless_database_pool.rb +2 -0
- data/test/adapters/spatialite.rb +2 -0
- data/test/adapters/sqlite3.rb +2 -0
- data/test/adapters/trilogy.rb +9 -0
- data/test/database.yml.sample +7 -0
- data/test/{travis → github}/database.yml +9 -3
- data/test/import_test.rb +108 -41
- data/test/jdbcmysql/import_test.rb +5 -3
- data/test/jdbcpostgresql/import_test.rb +4 -2
- data/test/jdbcsqlite3/import_test.rb +4 -2
- data/test/makara_postgis/import_test.rb +4 -2
- data/test/models/account.rb +2 -0
- data/test/models/alarm.rb +2 -0
- data/test/models/animal.rb +8 -0
- data/test/models/author.rb +9 -0
- data/test/models/bike_maker.rb +3 -0
- data/test/models/book.rb +12 -3
- data/test/models/car.rb +2 -0
- data/test/models/card.rb +5 -0
- data/test/models/chapter.rb +2 -0
- data/test/models/composite_book.rb +19 -0
- data/test/models/composite_chapter.rb +12 -0
- data/test/models/customer.rb +18 -0
- data/test/models/deck.rb +8 -0
- data/test/models/dictionary.rb +2 -0
- data/test/models/discount.rb +2 -0
- data/test/models/end_note.rb +2 -0
- data/test/models/group.rb +2 -0
- data/test/models/order.rb +17 -0
- data/test/models/playing_card.rb +4 -0
- data/test/models/promotion.rb +2 -0
- data/test/models/question.rb +2 -0
- data/test/models/rule.rb +2 -0
- data/test/models/tag.rb +9 -1
- data/test/models/tag_alias.rb +11 -0
- data/test/models/topic.rb +8 -0
- data/test/models/user.rb +2 -0
- data/test/models/user_token.rb +2 -0
- data/test/models/vendor.rb +2 -0
- data/test/models/widget.rb +12 -3
- data/test/mysql2/import_test.rb +5 -3
- data/test/mysql2_makara/import_test.rb +5 -3
- data/test/mysqlspatial2/import_test.rb +5 -3
- data/test/postgis/import_test.rb +4 -2
- data/test/postgresql/import_test.rb +4 -2
- data/test/schema/generic_schema.rb +37 -1
- data/test/schema/jdbcpostgresql_schema.rb +3 -1
- data/test/schema/mysql2_schema.rb +2 -0
- data/test/schema/postgis_schema.rb +3 -1
- data/test/schema/postgresql_schema.rb +38 -4
- data/test/schema/sqlite3_schema.rb +2 -0
- data/test/schema/version.rb +2 -0
- data/test/sqlite3/import_test.rb +4 -2
- data/test/support/active_support/test_case_extensions.rb +3 -5
- data/test/support/assertions.rb +2 -0
- data/test/support/factories.rb +2 -0
- data/test/support/generate.rb +4 -2
- data/test/support/mysql/import_examples.rb +7 -8
- data/test/support/postgresql/import_examples.rb +121 -53
- data/test/support/shared_examples/on_duplicate_key_ignore.rb +2 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +69 -10
- data/test/support/shared_examples/recursive_import.rb +137 -1
- data/test/support/sqlite3/import_examples.rb +2 -1
- data/test/synchronize_test.rb +2 -0
- data/test/test_helper.rb +38 -24
- data/test/trilogy/import_test.rb +7 -0
- data/test/value_sets_bytes_parser_test.rb +3 -1
- data/test/value_sets_records_parser_test.rb +3 -1
- metadata +46 -22
- data/.travis.yml +0 -74
- data/gemfiles/3.2.gemfile +0 -2
- data/gemfiles/4.0.gemfile +0 -2
- data/gemfiles/4.1.gemfile +0 -2
- data/gemfiles/4.2.gemfile +0 -2
- data/gemfiles/5.0.gemfile +0 -2
- data/gemfiles/5.1.gemfile +0 -2
- data/lib/activerecord-import/mysql2.rb +0 -7
- data/lib/activerecord-import/postgresql.rb +0 -7
- data/lib/activerecord-import/sqlite3.rb +0 -7
data/LICENSE
CHANGED
|
@@ -1,56 +1,21 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
d) make other distribution arrangements with the author.
|
|
24
|
-
|
|
25
|
-
3. You may distribute the software in object code or binary form,
|
|
26
|
-
provided that you do at least ONE of the following:
|
|
27
|
-
|
|
28
|
-
a) distribute the binaries and library files of the software,
|
|
29
|
-
together with instructions (in the manual page or equivalent)
|
|
30
|
-
on where to get the original distribution.
|
|
31
|
-
|
|
32
|
-
b) accompany the distribution with the machine-readable source of
|
|
33
|
-
the software.
|
|
34
|
-
|
|
35
|
-
c) give non-standard binaries non-standard names, with
|
|
36
|
-
instructions on where to get the original software distribution.
|
|
37
|
-
|
|
38
|
-
d) make other distribution arrangements with the author.
|
|
39
|
-
|
|
40
|
-
4. You may modify and include the part of the software into any other
|
|
41
|
-
software (possibly commercial). But some files in the distribution
|
|
42
|
-
are not written by the author, so that they are not under these terms.
|
|
43
|
-
|
|
44
|
-
For the list of those files and their copying conditions, see the
|
|
45
|
-
file LEGAL.
|
|
46
|
-
|
|
47
|
-
5. The scripts and library files supplied as input to or produced as
|
|
48
|
-
output from the software do not automatically fall under the
|
|
49
|
-
copyright of the software, but belong to whomever generated them,
|
|
50
|
-
and may be sold commercially, and may be aggregated with this
|
|
51
|
-
software.
|
|
52
|
-
|
|
53
|
-
6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
|
|
54
|
-
IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
|
|
55
|
-
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
56
|
-
PURPOSE.
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2020 Zach Dennis <zach.dennis@gmail.com>
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.markdown
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Activerecord-Import
|
|
1
|
+
# Activerecord-Import 
|
|
2
2
|
|
|
3
3
|
Activerecord-Import is a library for bulk inserting data using ActiveRecord.
|
|
4
4
|
|
|
@@ -60,6 +60,8 @@ The gem provides the following high-level features:
|
|
|
60
60
|
|
|
61
61
|
#### Introduction
|
|
62
62
|
|
|
63
|
+
This gem adds an `import` method (or `bulk_import`, for compatibility with gems like `elasticsearch-model`; see [Conflicts With Other Gems](#conflicts-with-other-gems)) to ActiveRecord classes.
|
|
64
|
+
|
|
63
65
|
Without `activerecord-import`, you'd write something like this:
|
|
64
66
|
|
|
65
67
|
```ruby
|
|
@@ -229,9 +231,22 @@ columns = [ :title ]
|
|
|
229
231
|
Book.import columns, books, batch_size: 2
|
|
230
232
|
```
|
|
231
233
|
|
|
234
|
+
If your import is particularly large or slow (possibly due to [callbacks](#callbacks)) whilst batch importing, you might want a way to report back on progress. This is supported by passing a callable as the `batch_progress` option. e.g:
|
|
235
|
+
|
|
236
|
+
```ruby
|
|
237
|
+
my_proc = ->(rows_size, num_batches, current_batch_number, batch_duration_in_secs) {
|
|
238
|
+
# Using the arguments provided to the callable, you can
|
|
239
|
+
# send an email, post to a websocket,
|
|
240
|
+
# update slack, alert if import is taking too long, etc.
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
Book.import columns, books, batch_size: 2, batch_progress: my_proc
|
|
244
|
+
```
|
|
245
|
+
|
|
232
246
|
#### Recursive
|
|
233
247
|
|
|
234
|
-
|
|
248
|
+
> **Note**
|
|
249
|
+
> This only works with PostgreSQL and ActiveRecord objects. This won't work with hashes or arrays as recursive inputs.
|
|
235
250
|
|
|
236
251
|
Assume that Books <code>has_many</code> Reviews.
|
|
237
252
|
|
|
@@ -247,20 +262,22 @@ Book.import books, recursive: true
|
|
|
247
262
|
|
|
248
263
|
### Options
|
|
249
264
|
|
|
250
|
-
Key
|
|
251
|
-
|
|
252
|
-
:validate
|
|
253
|
-
:validate_uniqueness
|
|
254
|
-
:validate_with_context
|
|
255
|
-
:
|
|
256
|
-
:
|
|
257
|
-
:
|
|
258
|
-
:
|
|
259
|
-
:
|
|
260
|
-
:
|
|
261
|
-
:
|
|
262
|
-
:
|
|
263
|
-
:
|
|
265
|
+
Key | Options | Default | Description
|
|
266
|
+
------------------------- | --------------------- | ------------------ | -----------
|
|
267
|
+
:validate | `true`/`false` | `true` | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
|
|
268
|
+
:validate_uniqueness | `true`/`false` | `false` | Whether or not to run ActiveRecord uniqueness validations. Beware this will incur an sql query per-record (N+1 queries). (requires `>= v0.27.0`).
|
|
269
|
+
:validate_with_context | `Symbol` |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
|
|
270
|
+
:track_validation_failures| `true`/`false` | `false` | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]`
|
|
271
|
+
:on_duplicate_key_ignore | `true`/`false` | `false` | Allows skipping records with duplicate keys. See [here](#duplicate-key-ignore) for more details.
|
|
272
|
+
:ignore | `true`/`false` | `false` | Alias for :on_duplicate_key_ignore.
|
|
273
|
+
:on_duplicate_key_update | :all, `Array`, `Hash` | N/A | Allows upsert logic to be used. See [here](#duplicate-key-update) for more details.
|
|
274
|
+
:synchronize | `Array` | N/A | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
|
|
275
|
+
:timestamps | `true`/`false` | `true` | Enables/disables timestamps on imported records.
|
|
276
|
+
:recursive | `true`/`false` | `false` | Imports has_many/has_one associations (PostgreSQL only).
|
|
277
|
+
:recursive_on_duplicate_key_update | `Hash` | N/A | Allows upsert logic to be used for recursive associations. The hash key is the association name and the value has the same options as `:on_duplicate_key_update`. See [here](#duplicate-key-update) for more details.
|
|
278
|
+
:batch_size | `Integer` | total # of records | Max number of records to insert per import
|
|
279
|
+
:raise_error | `true`/`false` | `false` | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
|
|
280
|
+
:all_or_none | `true`/`false` | `false` | Will not import any records if there is a record with validation errors.
|
|
264
281
|
|
|
265
282
|
#### Duplicate Key Ignore
|
|
266
283
|
|
|
@@ -348,6 +365,29 @@ book.reload.title # => "Book1" (stayed the same)
|
|
|
348
365
|
book.reload.author # => "Bob Barker" (changed)
|
|
349
366
|
```
|
|
350
367
|
|
|
368
|
+
PostgreSQL Using partial indexes
|
|
369
|
+
|
|
370
|
+
```ruby
|
|
371
|
+
book = Book.create! title: "Book1", author: "George Orwell", published_at: Time.now
|
|
372
|
+
book.author = "Bob Barker"
|
|
373
|
+
|
|
374
|
+
# in migration
|
|
375
|
+
execute <<-SQL
|
|
376
|
+
CREATE INDEX books_published_at_index ON books (published_at) WHERE published_at IS NOT NULL;
|
|
377
|
+
SQL
|
|
378
|
+
|
|
379
|
+
# PostgreSQL version
|
|
380
|
+
Book.import [book], on_duplicate_key_update: {
|
|
381
|
+
conflict_target: [:id],
|
|
382
|
+
index_predicate: "published_at IS NOT NULL",
|
|
383
|
+
columns: [:author]
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
book.reload.title # => "Book1" (stayed the same)
|
|
387
|
+
book.reload.author # => "Bob Barker" (changed)
|
|
388
|
+
book.reload.published_at # => 2017-10-09 (stayed the same)
|
|
389
|
+
```
|
|
390
|
+
|
|
351
391
|
PostgreSQL Using constraints
|
|
352
392
|
|
|
353
393
|
```ruby
|
|
@@ -376,7 +416,7 @@ Book.import books, validate_uniqueness: true
|
|
|
376
416
|
|
|
377
417
|
### Return Info
|
|
378
418
|
|
|
379
|
-
The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed
|
|
419
|
+
The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed.
|
|
380
420
|
|
|
381
421
|
```ruby
|
|
382
422
|
articles = [
|
|
@@ -417,7 +457,9 @@ Should you wish to specify those columns, you may use the option `timestamps: fa
|
|
|
417
457
|
|
|
418
458
|
However, it is also possible to set just `:created_at` in specific records. In this case despite using `timestamps: true`, `:created_at` will be updated only in records where that field is `nil`. Same rule applies for record associations when enabling the option `recursive: true`.
|
|
419
459
|
|
|
420
|
-
If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is
|
|
460
|
+
If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is.
|
|
461
|
+
> **Note**
|
|
462
|
+
> If you are using ActiveRecord 7.0 or later, please use `ActiveRecord.default_timezone` instead.
|
|
421
463
|
|
|
422
464
|
### Callbacks
|
|
423
465
|
|
|
@@ -489,7 +531,8 @@ This allows an external gem to dynamically add an adapter without the need to ad
|
|
|
489
531
|
|
|
490
532
|
### Requiring
|
|
491
533
|
|
|
492
|
-
Note
|
|
534
|
+
> **Note**
|
|
535
|
+
> These instructions will only work if you are using version 0.2.0 or higher.
|
|
493
536
|
|
|
494
537
|
#### Autoloading via Bundler
|
|
495
538
|
|
|
@@ -526,11 +569,11 @@ require 'activerecord-import'
|
|
|
526
569
|
### Load Path Setup
|
|
527
570
|
To understand how rubygems loads code you can reference the following:
|
|
528
571
|
|
|
529
|
-
|
|
572
|
+
https://guides.rubygems.org/patterns/#loading-code
|
|
530
573
|
|
|
531
574
|
And an example of how active_record dynamically load adapters:
|
|
532
575
|
|
|
533
|
-
https://github.com/rails/rails/blob/
|
|
576
|
+
https://github.com/rails/rails/blob/main/activerecord/lib/active_record/connection_adapters.rb
|
|
534
577
|
|
|
535
578
|
In summary, when a gem is loaded rubygems adds the `lib` folder of the gem to the global load path `$LOAD_PATH` so that all `require` lookups will not propagate through all of the folders on the load path. When a `require` is issued each folder on the `$LOAD_PATH` is checked for the file and/or folder referenced. This allows a gem (like activerecord-import) to define push the activerecord-import folder (or namespace) on the `$LOAD_PATH` and any adapters provided by activerecord-import will be found by rubygems when the require is issued.
|
|
536
579
|
|
|
@@ -556,7 +599,7 @@ When rubygems pushes the `lib` folder onto the load path a `require` will now fi
|
|
|
556
599
|
|
|
557
600
|
Activerecord-Import adds the `.import` method onto `ActiveRecord::Base`. There are other gems, such as `elasticsearch-rails`, that do the same thing. In conflicts such as this, there is an aliased method named `.bulk_import` that can be used interchangeably.
|
|
558
601
|
|
|
559
|
-
If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by
|
|
602
|
+
If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by explicitly setting this value within the model. For example:
|
|
560
603
|
|
|
561
604
|
```ruby
|
|
562
605
|
class Post < ActiveRecord::Base
|
|
@@ -599,13 +642,26 @@ After that, you can run the tests. They run against multiple tests and ActiveRec
|
|
|
599
642
|
|
|
600
643
|
This is one example of how to run the tests:
|
|
601
644
|
|
|
602
|
-
```
|
|
645
|
+
```bash
|
|
603
646
|
rm Gemfile.lock
|
|
604
|
-
AR_VERSION=
|
|
605
|
-
AR_VERSION=
|
|
647
|
+
AR_VERSION=7.0 bundle install
|
|
648
|
+
AR_VERSION=7.0 bundle exec rake test:postgresql test:sqlite3 test:mysql2
|
|
649
|
+
```
|
|
650
|
+
|
|
651
|
+
Once you have pushed up your changes, you can find your CI results [here](https://github.com/zdennis/activerecord-import/actions).
|
|
652
|
+
|
|
653
|
+
#### Docker Setup
|
|
654
|
+
|
|
655
|
+
Before you begin, make sure you have [Docker](https://www.docker.com/products/docker-desktop/) and [Docker Compose](https://docs.docker.com/compose/) installed on your machine. If you don't, you can install both via Homebrew using the following command:
|
|
656
|
+
|
|
657
|
+
```bash
|
|
658
|
+
brew install docker && brew install docker-compose
|
|
606
659
|
```
|
|
660
|
+
##### Steps
|
|
607
661
|
|
|
608
|
-
|
|
662
|
+
1. In your terminal run `docker-compose up --build`
|
|
663
|
+
1. In another tab/window run `docker-compose exec app bash`
|
|
664
|
+
1. In that same terminal run the mysql2 test by running `bundle exec rake test:mysql2`
|
|
609
665
|
|
|
610
666
|
## Issue Triage [](https://www.codetriage.com/zdennis/activerecord-import)
|
|
611
667
|
|
|
@@ -613,7 +669,7 @@ You can triage issues which may include reproducing bug reports or asking for vi
|
|
|
613
669
|
|
|
614
670
|
# License
|
|
615
671
|
|
|
616
|
-
This is licensed under the
|
|
672
|
+
This is licensed under the MIT license.
|
|
617
673
|
|
|
618
674
|
# Author
|
|
619
675
|
|
data/Rakefile
CHANGED
data/activerecord-import.gemspec
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
#
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
2
3
|
require File.expand_path('../lib/activerecord-import/version', __FILE__)
|
|
3
4
|
|
|
4
5
|
Gem::Specification.new do |gem|
|
|
@@ -6,8 +7,12 @@ Gem::Specification.new do |gem|
|
|
|
6
7
|
gem.email = ["zach.dennis@gmail.com"]
|
|
7
8
|
gem.summary = "Bulk insert extension for ActiveRecord"
|
|
8
9
|
gem.description = "A library for bulk inserting data using ActiveRecord."
|
|
9
|
-
gem.homepage = "
|
|
10
|
-
gem.license = "
|
|
10
|
+
gem.homepage = "https://github.com/zdennis/activerecord-import"
|
|
11
|
+
gem.license = "MIT"
|
|
12
|
+
|
|
13
|
+
gem.metadata = {
|
|
14
|
+
"changelog_uri" => "https://github.com/zdennis/activerecord-import/blob/master/CHANGELOG.md"
|
|
15
|
+
}
|
|
11
16
|
|
|
12
17
|
gem.files = `git ls-files`.split($\)
|
|
13
18
|
gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
|
|
@@ -16,8 +21,8 @@ Gem::Specification.new do |gem|
|
|
|
16
21
|
gem.require_paths = ["lib"]
|
|
17
22
|
gem.version = ActiveRecord::Import::VERSION
|
|
18
23
|
|
|
19
|
-
gem.required_ruby_version = ">=
|
|
24
|
+
gem.required_ruby_version = ">= 2.4.0"
|
|
20
25
|
|
|
21
|
-
gem.add_runtime_dependency "activerecord", ">=
|
|
26
|
+
gem.add_runtime_dependency "activerecord", ">= 4.2"
|
|
22
27
|
gem.add_development_dependency "rake"
|
|
23
28
|
end
|
data/benchmarks/benchmark.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'pathname'
|
|
2
4
|
require "fileutils"
|
|
3
5
|
require "active_record"
|
|
@@ -20,7 +22,11 @@ FileUtils.mkdir_p 'log'
|
|
|
20
22
|
ActiveRecord::Base.configurations["test"] = YAML.load_file(File.join(benchmark_dir, "../test/database.yml"))[options.adapter]
|
|
21
23
|
ActiveRecord::Base.logger = Logger.new("log/test.log")
|
|
22
24
|
ActiveRecord::Base.logger.level = Logger::DEBUG
|
|
23
|
-
ActiveRecord
|
|
25
|
+
if ActiveRecord.respond_to?(:default_timezone)
|
|
26
|
+
ActiveRecord.default_timezone = :utc
|
|
27
|
+
else
|
|
28
|
+
ActiveRecord::Base.default_timezone = :utc
|
|
29
|
+
end
|
|
24
30
|
|
|
25
31
|
require "activerecord-import"
|
|
26
32
|
ActiveRecord::Base.establish_connection(:test)
|
|
@@ -35,11 +41,9 @@ require File.join(benchmark_dir, "../test/schema/generic_schema")
|
|
|
35
41
|
adapter_schema = File.join(benchmark_dir, "schema/#{options.adapter}_schema.rb")
|
|
36
42
|
require adapter_schema if File.exist?(adapter_schema)
|
|
37
43
|
|
|
38
|
-
Dir[File.dirname(__FILE__)
|
|
44
|
+
Dir["#{File.dirname(__FILE__)}/models/*.rb"].sort.each { |file| require file }
|
|
39
45
|
|
|
40
46
|
require File.join( benchmark_dir, 'lib', "#{options.adapter}_benchmark" )
|
|
41
|
-
|
|
42
|
-
table_types = nil
|
|
43
47
|
table_types = if options.benchmark_all_types
|
|
44
48
|
["all"]
|
|
45
49
|
else
|
|
@@ -47,8 +51,8 @@ else
|
|
|
47
51
|
end
|
|
48
52
|
|
|
49
53
|
letter = options.adapter[0].chr
|
|
50
|
-
clazz_str = letter.upcase + options.adapter[1
|
|
51
|
-
clazz = Object.const_get( clazz_str
|
|
54
|
+
clazz_str = letter.upcase + options.adapter[1..].downcase
|
|
55
|
+
clazz = Object.const_get( "#{clazz_str}Benchmark" )
|
|
52
56
|
|
|
53
57
|
benchmarks = []
|
|
54
58
|
options.number_of_objects.each do |num|
|
data/benchmarks/lib/base.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
class BenchmarkBase
|
|
2
4
|
attr_reader :results
|
|
3
5
|
|
|
@@ -14,7 +16,7 @@ class BenchmarkBase
|
|
|
14
16
|
end
|
|
15
17
|
end
|
|
16
18
|
|
|
17
|
-
# Returns
|
|
19
|
+
# Returns a struct which contains two attritues, +description+ and +tms+ after performing an
|
|
18
20
|
# actual benchmark.
|
|
19
21
|
#
|
|
20
22
|
# == PARAMETERS
|
|
@@ -22,18 +24,21 @@ class BenchmarkBase
|
|
|
22
24
|
# * blk - the block of code to benchmark
|
|
23
25
|
#
|
|
24
26
|
# == RETURNS
|
|
25
|
-
#
|
|
27
|
+
# A struct object with the following attributes:
|
|
26
28
|
# * description - the description of the benchmark ran
|
|
27
29
|
# * tms - a Benchmark::Tms containing the results of the benchmark
|
|
28
|
-
|
|
30
|
+
|
|
31
|
+
BmStruct = Struct.new( :description, :tms, :failed, keyword_init: true )
|
|
32
|
+
|
|
33
|
+
def bm( description, &block )
|
|
29
34
|
tms = nil
|
|
30
35
|
puts "Benchmarking #{description}"
|
|
31
36
|
|
|
32
|
-
Benchmark.bm { |x| tms = x.report
|
|
37
|
+
Benchmark.bm { |x| tms = x.report(&block) }
|
|
33
38
|
delete_all
|
|
34
39
|
failed = false
|
|
35
40
|
|
|
36
|
-
|
|
41
|
+
BmStruct.new( description: description, tms: tms, failed: failed )
|
|
37
42
|
end
|
|
38
43
|
|
|
39
44
|
# Given a model class (ie: Topic), and an array of columns and value sets
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'optparse'
|
|
2
|
-
require 'ostruct'
|
|
3
4
|
|
|
4
5
|
#
|
|
5
6
|
# == PARAMETERS
|
|
@@ -27,7 +28,7 @@ module BenchmarkOptionParser
|
|
|
27
28
|
print_valid_table_types( options, prefix: " " )
|
|
28
29
|
end
|
|
29
30
|
|
|
30
|
-
# TODO IMPLEMENT THIS
|
|
31
|
+
# TODO: IMPLEMENT THIS
|
|
31
32
|
def self.print_valid_table_types( options, hsh = { prefix: '' } )
|
|
32
33
|
if !options.table_types.keys.empty?
|
|
33
34
|
options.table_types.keys.sort.each { |type| puts hsh[:prefix].to_s + type.to_s }
|
|
@@ -36,8 +37,11 @@ module BenchmarkOptionParser
|
|
|
36
37
|
end
|
|
37
38
|
end
|
|
38
39
|
|
|
40
|
+
OptionsStruct = Struct.new( :adapter, :table_types, :delete_on_finish, :number_of_objects, :outputs,
|
|
41
|
+
:benchmark_all_types, keyword_init: true )
|
|
42
|
+
OutputStruct = Struct.new( :format, :filename, keyword_init: true )
|
|
39
43
|
def self.parse( args )
|
|
40
|
-
options =
|
|
44
|
+
options = OptionsStruct.new(
|
|
41
45
|
adapter: 'mysql2',
|
|
42
46
|
table_types: {},
|
|
43
47
|
delete_on_finish: true,
|
|
@@ -79,12 +83,12 @@ module BenchmarkOptionParser
|
|
|
79
83
|
|
|
80
84
|
# print results in CSV format
|
|
81
85
|
opts.on( "--to-csv [String]", "Print results in a CSV file format" ) do |filename|
|
|
82
|
-
options.outputs <<
|
|
86
|
+
options.outputs << OutputStruct.new( format: 'csv', filename: filename)
|
|
83
87
|
end
|
|
84
88
|
|
|
85
89
|
# print results in HTML format
|
|
86
90
|
opts.on( "--to-html [String]", "Print results in HTML format" ) do |filename|
|
|
87
|
-
options.outputs <<
|
|
91
|
+
options.outputs << OutputStruct.new( format: 'html', filename: filename )
|
|
88
92
|
end
|
|
89
93
|
end # end opt.parse!
|
|
90
94
|
|
|
@@ -98,7 +102,7 @@ module BenchmarkOptionParser
|
|
|
98
102
|
end
|
|
99
103
|
|
|
100
104
|
options.number_of_objects = [1000] if options.number_of_objects.empty?
|
|
101
|
-
options.outputs = [
|
|
105
|
+
options.outputs = [OutputStruct.new( format: 'html', filename: 'benchmark.html')] if options.outputs.empty?
|
|
102
106
|
|
|
103
107
|
print_options( options )
|
|
104
108
|
|
data/benchmarks/lib/float.rb
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'erb'
|
|
2
4
|
|
|
3
5
|
module OutputToHTML
|
|
4
|
-
TEMPLATE_HEADER = <<"EOT"
|
|
6
|
+
TEMPLATE_HEADER = <<"EOT"
|
|
5
7
|
<div>
|
|
6
8
|
All times are rounded to the nearest thousandth for display purposes. Speedups next to each time are computed
|
|
7
9
|
before any rounding occurs. Also, all speedup calculations are computed by comparing a given time against
|
|
@@ -9,7 +11,7 @@ module OutputToHTML
|
|
|
9
11
|
</div>
|
|
10
12
|
EOT
|
|
11
13
|
|
|
12
|
-
TEMPLATE = <<"EOT"
|
|
14
|
+
TEMPLATE = <<"EOT"
|
|
13
15
|
<style>
|
|
14
16
|
td#benchmarkTitle {
|
|
15
17
|
border: 1px solid black;
|
data/docker-compose.yml
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
version: "3.5"
|
|
2
|
+
|
|
3
|
+
services:
|
|
4
|
+
mysql:
|
|
5
|
+
platform: linux/x86_64
|
|
6
|
+
image: mysql:5.7
|
|
7
|
+
volumes:
|
|
8
|
+
- mysql-data:/var/lib/mysql
|
|
9
|
+
ports:
|
|
10
|
+
- "3306:3306"
|
|
11
|
+
|
|
12
|
+
postgresql:
|
|
13
|
+
image: postgres:latest
|
|
14
|
+
volumes:
|
|
15
|
+
- postgresql-data:/var/lib/postgresql/data
|
|
16
|
+
ports:
|
|
17
|
+
- "5432:5432"
|
|
18
|
+
|
|
19
|
+
app:
|
|
20
|
+
build:
|
|
21
|
+
context: .
|
|
22
|
+
environment:
|
|
23
|
+
DB_HOST: mysql
|
|
24
|
+
AR_VERSION: 7.0
|
|
25
|
+
volumes:
|
|
26
|
+
- .:/usr/src/app
|
|
27
|
+
depends_on:
|
|
28
|
+
- mysql
|
|
29
|
+
- postgresql
|
|
30
|
+
command: tail -f /dev/null
|
|
31
|
+
|
|
32
|
+
volumes:
|
|
33
|
+
mysql-data:
|
|
34
|
+
postgresql-data:
|
data/gemfiles/5.2.gemfile
CHANGED
data/gemfiles/6.0.gemfile
CHANGED
data/gemfiles/6.1.gemfile
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
require "activerecord-import/adapters/mysql_adapter"
|
|
1
|
+
# frozen_string_literal: true
|
|
3
2
|
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
require "active_record/connection_adapters/mysql2_adapter"
|
|
4
|
+
require "activerecord-import/adapters/mysql2_adapter"
|
|
5
|
+
|
|
6
|
+
class ActiveRecord::ConnectionAdapters::Mysql2Adapter
|
|
7
|
+
include ActiveRecord::Import::Mysql2Adapter
|
|
6
8
|
end
|