activerecord-import 1.0.4 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yaml +159 -0
- data/.gitignore +5 -0
- data/.rubocop.yml +76 -7
- data/.rubocop_todo.yml +10 -16
- data/Brewfile +3 -1
- data/CHANGELOG.md +143 -3
- data/Dockerfile +23 -0
- data/Gemfile +28 -24
- data/LICENSE +21 -56
- data/README.markdown +83 -27
- data/Rakefile +3 -0
- data/activerecord-import.gemspec +10 -5
- data/benchmarks/benchmark.rb +10 -6
- data/benchmarks/lib/base.rb +10 -5
- data/benchmarks/lib/cli_parser.rb +10 -6
- data/benchmarks/lib/float.rb +2 -0
- data/benchmarks/lib/mysql2_benchmark.rb +2 -0
- data/benchmarks/lib/output_to_csv.rb +2 -0
- data/benchmarks/lib/output_to_html.rb +4 -2
- data/benchmarks/models/test_innodb.rb +2 -0
- data/benchmarks/models/test_memory.rb +2 -0
- data/benchmarks/models/test_myisam.rb +2 -0
- data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
- data/docker-compose.yml +34 -0
- data/gemfiles/5.2.gemfile +2 -0
- data/gemfiles/6.0.gemfile +3 -0
- data/gemfiles/6.1.gemfile +4 -1
- data/gemfiles/7.0.gemfile +4 -0
- data/gemfiles/7.1.gemfile +3 -0
- data/gemfiles/7.2.gemfile +3 -0
- data/gemfiles/8.0.gemfile +3 -0
- data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
- data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/trilogy_adapter.rb +8 -0
- data/lib/activerecord-import/adapters/abstract_adapter.rb +9 -6
- data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +30 -21
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +68 -48
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +37 -30
- data/lib/activerecord-import/adapters/trilogy_adapter.rb +7 -0
- data/lib/activerecord-import/base.rb +3 -1
- data/lib/activerecord-import/import.rb +160 -58
- data/lib/activerecord-import/synchronize.rb +3 -1
- data/lib/activerecord-import/value_sets_parser.rb +5 -0
- data/lib/activerecord-import/version.rb +3 -1
- data/lib/activerecord-import.rb +2 -1
- data/test/adapters/jdbcmysql.rb +2 -0
- data/test/adapters/jdbcpostgresql.rb +2 -0
- data/test/adapters/jdbcsqlite3.rb +2 -0
- data/test/adapters/makara_postgis.rb +2 -0
- data/test/adapters/mysql2.rb +2 -0
- data/test/adapters/mysql2_makara.rb +2 -0
- data/test/adapters/mysql2spatial.rb +2 -0
- data/test/adapters/postgis.rb +2 -0
- data/test/adapters/postgresql.rb +2 -0
- data/test/adapters/postgresql_makara.rb +2 -0
- data/test/adapters/seamless_database_pool.rb +2 -0
- data/test/adapters/spatialite.rb +2 -0
- data/test/adapters/sqlite3.rb +2 -0
- data/test/adapters/trilogy.rb +9 -0
- data/test/database.yml.sample +7 -0
- data/test/{travis → github}/database.yml +9 -3
- data/test/import_test.rb +108 -41
- data/test/jdbcmysql/import_test.rb +5 -3
- data/test/jdbcpostgresql/import_test.rb +4 -2
- data/test/jdbcsqlite3/import_test.rb +4 -2
- data/test/makara_postgis/import_test.rb +4 -2
- data/test/models/account.rb +2 -0
- data/test/models/alarm.rb +2 -0
- data/test/models/animal.rb +8 -0
- data/test/models/author.rb +9 -0
- data/test/models/bike_maker.rb +3 -0
- data/test/models/book.rb +12 -3
- data/test/models/car.rb +2 -0
- data/test/models/card.rb +5 -0
- data/test/models/chapter.rb +2 -0
- data/test/models/composite_book.rb +19 -0
- data/test/models/composite_chapter.rb +12 -0
- data/test/models/customer.rb +18 -0
- data/test/models/deck.rb +8 -0
- data/test/models/dictionary.rb +2 -0
- data/test/models/discount.rb +2 -0
- data/test/models/end_note.rb +2 -0
- data/test/models/group.rb +2 -0
- data/test/models/order.rb +17 -0
- data/test/models/playing_card.rb +4 -0
- data/test/models/promotion.rb +2 -0
- data/test/models/question.rb +2 -0
- data/test/models/rule.rb +2 -0
- data/test/models/tag.rb +9 -1
- data/test/models/tag_alias.rb +11 -0
- data/test/models/topic.rb +8 -0
- data/test/models/user.rb +2 -0
- data/test/models/user_token.rb +2 -0
- data/test/models/vendor.rb +2 -0
- data/test/models/widget.rb +12 -3
- data/test/mysql2/import_test.rb +5 -3
- data/test/mysql2_makara/import_test.rb +5 -3
- data/test/mysqlspatial2/import_test.rb +5 -3
- data/test/postgis/import_test.rb +4 -2
- data/test/postgresql/import_test.rb +4 -2
- data/test/schema/generic_schema.rb +37 -1
- data/test/schema/jdbcpostgresql_schema.rb +3 -1
- data/test/schema/mysql2_schema.rb +2 -0
- data/test/schema/postgis_schema.rb +3 -1
- data/test/schema/postgresql_schema.rb +38 -4
- data/test/schema/sqlite3_schema.rb +2 -0
- data/test/schema/version.rb +2 -0
- data/test/sqlite3/import_test.rb +4 -2
- data/test/support/active_support/test_case_extensions.rb +3 -5
- data/test/support/assertions.rb +2 -0
- data/test/support/factories.rb +2 -0
- data/test/support/generate.rb +4 -2
- data/test/support/mysql/import_examples.rb +7 -8
- data/test/support/postgresql/import_examples.rb +121 -53
- data/test/support/shared_examples/on_duplicate_key_ignore.rb +2 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +69 -10
- data/test/support/shared_examples/recursive_import.rb +137 -1
- data/test/support/sqlite3/import_examples.rb +2 -1
- data/test/synchronize_test.rb +2 -0
- data/test/test_helper.rb +38 -24
- data/test/trilogy/import_test.rb +7 -0
- data/test/value_sets_bytes_parser_test.rb +3 -1
- data/test/value_sets_records_parser_test.rb +3 -1
- metadata +46 -22
- data/.travis.yml +0 -74
- data/gemfiles/3.2.gemfile +0 -2
- data/gemfiles/4.0.gemfile +0 -2
- data/gemfiles/4.1.gemfile +0 -2
- data/gemfiles/4.2.gemfile +0 -2
- data/gemfiles/5.0.gemfile +0 -2
- data/gemfiles/5.1.gemfile +0 -2
- data/lib/activerecord-import/mysql2.rb +0 -7
- data/lib/activerecord-import/postgresql.rb +0 -7
- data/lib/activerecord-import/sqlite3.rb +0 -7
data/LICENSE
CHANGED
@@ -1,56 +1,21 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
d) make other distribution arrangements with the author.
|
24
|
-
|
25
|
-
3. You may distribute the software in object code or binary form,
|
26
|
-
provided that you do at least ONE of the following:
|
27
|
-
|
28
|
-
a) distribute the binaries and library files of the software,
|
29
|
-
together with instructions (in the manual page or equivalent)
|
30
|
-
on where to get the original distribution.
|
31
|
-
|
32
|
-
b) accompany the distribution with the machine-readable source of
|
33
|
-
the software.
|
34
|
-
|
35
|
-
c) give non-standard binaries non-standard names, with
|
36
|
-
instructions on where to get the original software distribution.
|
37
|
-
|
38
|
-
d) make other distribution arrangements with the author.
|
39
|
-
|
40
|
-
4. You may modify and include the part of the software into any other
|
41
|
-
software (possibly commercial). But some files in the distribution
|
42
|
-
are not written by the author, so that they are not under these terms.
|
43
|
-
|
44
|
-
For the list of those files and their copying conditions, see the
|
45
|
-
file LEGAL.
|
46
|
-
|
47
|
-
5. The scripts and library files supplied as input to or produced as
|
48
|
-
output from the software do not automatically fall under the
|
49
|
-
copyright of the software, but belong to whomever generated them,
|
50
|
-
and may be sold commercially, and may be aggregated with this
|
51
|
-
software.
|
52
|
-
|
53
|
-
6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
|
54
|
-
IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
|
55
|
-
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
56
|
-
PURPOSE.
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2020 Zach Dennis <zach.dennis@gmail.com>
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.markdown
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Activerecord-Import
|
1
|
+
# Activerecord-Import ![Build Status](https://github.com/zdennis/activerecord-import/actions/workflows/test.yaml/badge.svg)
|
2
2
|
|
3
3
|
Activerecord-Import is a library for bulk inserting data using ActiveRecord.
|
4
4
|
|
@@ -60,6 +60,8 @@ The gem provides the following high-level features:
|
|
60
60
|
|
61
61
|
#### Introduction
|
62
62
|
|
63
|
+
This gem adds an `import` method (or `bulk_import`, for compatibility with gems like `elasticsearch-model`; see [Conflicts With Other Gems](#conflicts-with-other-gems)) to ActiveRecord classes.
|
64
|
+
|
63
65
|
Without `activerecord-import`, you'd write something like this:
|
64
66
|
|
65
67
|
```ruby
|
@@ -229,9 +231,22 @@ columns = [ :title ]
|
|
229
231
|
Book.import columns, books, batch_size: 2
|
230
232
|
```
|
231
233
|
|
234
|
+
If your import is particularly large or slow (possibly due to [callbacks](#callbacks)) whilst batch importing, you might want a way to report back on progress. This is supported by passing a callable as the `batch_progress` option. e.g:
|
235
|
+
|
236
|
+
```ruby
|
237
|
+
my_proc = ->(rows_size, num_batches, current_batch_number, batch_duration_in_secs) {
|
238
|
+
# Using the arguments provided to the callable, you can
|
239
|
+
# send an email, post to a websocket,
|
240
|
+
# update slack, alert if import is taking too long, etc.
|
241
|
+
}
|
242
|
+
|
243
|
+
Book.import columns, books, batch_size: 2, batch_progress: my_proc
|
244
|
+
```
|
245
|
+
|
232
246
|
#### Recursive
|
233
247
|
|
234
|
-
|
248
|
+
> **Note**
|
249
|
+
> This only works with PostgreSQL and ActiveRecord objects. This won't work with hashes or arrays as recursive inputs.
|
235
250
|
|
236
251
|
Assume that Books <code>has_many</code> Reviews.
|
237
252
|
|
@@ -247,20 +262,22 @@ Book.import books, recursive: true
|
|
247
262
|
|
248
263
|
### Options
|
249
264
|
|
250
|
-
Key
|
251
|
-
|
252
|
-
:validate
|
253
|
-
:validate_uniqueness
|
254
|
-
:validate_with_context
|
255
|
-
:
|
256
|
-
:
|
257
|
-
:
|
258
|
-
:
|
259
|
-
:
|
260
|
-
:
|
261
|
-
:
|
262
|
-
:
|
263
|
-
:
|
265
|
+
Key | Options | Default | Description
|
266
|
+
------------------------- | --------------------- | ------------------ | -----------
|
267
|
+
:validate | `true`/`false` | `true` | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
|
268
|
+
:validate_uniqueness | `true`/`false` | `false` | Whether or not to run ActiveRecord uniqueness validations. Beware this will incur an sql query per-record (N+1 queries). (requires `>= v0.27.0`).
|
269
|
+
:validate_with_context | `Symbol` |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
|
270
|
+
:track_validation_failures| `true`/`false` | `false` | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]`
|
271
|
+
:on_duplicate_key_ignore | `true`/`false` | `false` | Allows skipping records with duplicate keys. See [here](#duplicate-key-ignore) for more details.
|
272
|
+
:ignore | `true`/`false` | `false` | Alias for :on_duplicate_key_ignore.
|
273
|
+
:on_duplicate_key_update | :all, `Array`, `Hash` | N/A | Allows upsert logic to be used. See [here](#duplicate-key-update) for more details.
|
274
|
+
:synchronize | `Array` | N/A | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
|
275
|
+
:timestamps | `true`/`false` | `true` | Enables/disables timestamps on imported records.
|
276
|
+
:recursive | `true`/`false` | `false` | Imports has_many/has_one associations (PostgreSQL only).
|
277
|
+
:recursive_on_duplicate_key_update | `Hash` | N/A | Allows upsert logic to be used for recursive associations. The hash key is the association name and the value has the same options as `:on_duplicate_key_update`. See [here](#duplicate-key-update) for more details.
|
278
|
+
:batch_size | `Integer` | total # of records | Max number of records to insert per import
|
279
|
+
:raise_error | `true`/`false` | `false` | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
|
280
|
+
:all_or_none | `true`/`false` | `false` | Will not import any records if there is a record with validation errors.
|
264
281
|
|
265
282
|
#### Duplicate Key Ignore
|
266
283
|
|
@@ -348,6 +365,29 @@ book.reload.title # => "Book1" (stayed the same)
|
|
348
365
|
book.reload.author # => "Bob Barker" (changed)
|
349
366
|
```
|
350
367
|
|
368
|
+
PostgreSQL Using partial indexes
|
369
|
+
|
370
|
+
```ruby
|
371
|
+
book = Book.create! title: "Book1", author: "George Orwell", published_at: Time.now
|
372
|
+
book.author = "Bob Barker"
|
373
|
+
|
374
|
+
# in migration
|
375
|
+
execute <<-SQL
|
376
|
+
CREATE INDEX books_published_at_index ON books (published_at) WHERE published_at IS NOT NULL;
|
377
|
+
SQL
|
378
|
+
|
379
|
+
# PostgreSQL version
|
380
|
+
Book.import [book], on_duplicate_key_update: {
|
381
|
+
conflict_target: [:id],
|
382
|
+
index_predicate: "published_at IS NOT NULL",
|
383
|
+
columns: [:author]
|
384
|
+
}
|
385
|
+
|
386
|
+
book.reload.title # => "Book1" (stayed the same)
|
387
|
+
book.reload.author # => "Bob Barker" (changed)
|
388
|
+
book.reload.published_at # => 2017-10-09 (stayed the same)
|
389
|
+
```
|
390
|
+
|
351
391
|
PostgreSQL Using constraints
|
352
392
|
|
353
393
|
```ruby
|
@@ -376,7 +416,7 @@ Book.import books, validate_uniqueness: true
|
|
376
416
|
|
377
417
|
### Return Info
|
378
418
|
|
379
|
-
The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed
|
419
|
+
The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed.
|
380
420
|
|
381
421
|
```ruby
|
382
422
|
articles = [
|
@@ -417,7 +457,9 @@ Should you wish to specify those columns, you may use the option `timestamps: fa
|
|
417
457
|
|
418
458
|
However, it is also possible to set just `:created_at` in specific records. In this case despite using `timestamps: true`, `:created_at` will be updated only in records where that field is `nil`. Same rule applies for record associations when enabling the option `recursive: true`.
|
419
459
|
|
420
|
-
If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is
|
460
|
+
If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is.
|
461
|
+
> **Note**
|
462
|
+
> If you are using ActiveRecord 7.0 or later, please use `ActiveRecord.default_timezone` instead.
|
421
463
|
|
422
464
|
### Callbacks
|
423
465
|
|
@@ -489,7 +531,8 @@ This allows an external gem to dynamically add an adapter without the need to ad
|
|
489
531
|
|
490
532
|
### Requiring
|
491
533
|
|
492
|
-
Note
|
534
|
+
> **Note**
|
535
|
+
> These instructions will only work if you are using version 0.2.0 or higher.
|
493
536
|
|
494
537
|
#### Autoloading via Bundler
|
495
538
|
|
@@ -526,11 +569,11 @@ require 'activerecord-import'
|
|
526
569
|
### Load Path Setup
|
527
570
|
To understand how rubygems loads code you can reference the following:
|
528
571
|
|
529
|
-
|
572
|
+
https://guides.rubygems.org/patterns/#loading-code
|
530
573
|
|
531
574
|
And an example of how active_record dynamically load adapters:
|
532
575
|
|
533
|
-
https://github.com/rails/rails/blob/
|
576
|
+
https://github.com/rails/rails/blob/main/activerecord/lib/active_record/connection_adapters.rb
|
534
577
|
|
535
578
|
In summary, when a gem is loaded rubygems adds the `lib` folder of the gem to the global load path `$LOAD_PATH` so that all `require` lookups will not propagate through all of the folders on the load path. When a `require` is issued each folder on the `$LOAD_PATH` is checked for the file and/or folder referenced. This allows a gem (like activerecord-import) to define push the activerecord-import folder (or namespace) on the `$LOAD_PATH` and any adapters provided by activerecord-import will be found by rubygems when the require is issued.
|
536
579
|
|
@@ -556,7 +599,7 @@ When rubygems pushes the `lib` folder onto the load path a `require` will now fi
|
|
556
599
|
|
557
600
|
Activerecord-Import adds the `.import` method onto `ActiveRecord::Base`. There are other gems, such as `elasticsearch-rails`, that do the same thing. In conflicts such as this, there is an aliased method named `.bulk_import` that can be used interchangeably.
|
558
601
|
|
559
|
-
If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by
|
602
|
+
If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by explicitly setting this value within the model. For example:
|
560
603
|
|
561
604
|
```ruby
|
562
605
|
class Post < ActiveRecord::Base
|
@@ -599,13 +642,26 @@ After that, you can run the tests. They run against multiple tests and ActiveRec
|
|
599
642
|
|
600
643
|
This is one example of how to run the tests:
|
601
644
|
|
602
|
-
```
|
645
|
+
```bash
|
603
646
|
rm Gemfile.lock
|
604
|
-
AR_VERSION=
|
605
|
-
AR_VERSION=
|
647
|
+
AR_VERSION=7.0 bundle install
|
648
|
+
AR_VERSION=7.0 bundle exec rake test:postgresql test:sqlite3 test:mysql2
|
649
|
+
```
|
650
|
+
|
651
|
+
Once you have pushed up your changes, you can find your CI results [here](https://github.com/zdennis/activerecord-import/actions).
|
652
|
+
|
653
|
+
#### Docker Setup
|
654
|
+
|
655
|
+
Before you begin, make sure you have [Docker](https://www.docker.com/products/docker-desktop/) and [Docker Compose](https://docs.docker.com/compose/) installed on your machine. If you don't, you can install both via Homebrew using the following command:
|
656
|
+
|
657
|
+
```bash
|
658
|
+
brew install docker && brew install docker-compose
|
606
659
|
```
|
660
|
+
##### Steps
|
607
661
|
|
608
|
-
|
662
|
+
1. In your terminal run `docker-compose up --build`
|
663
|
+
1. In another tab/window run `docker-compose exec app bash`
|
664
|
+
1. In that same terminal run the mysql2 test by running `bundle exec rake test:mysql2`
|
609
665
|
|
610
666
|
## Issue Triage [![Open Source Helpers](https://www.codetriage.com/zdennis/activerecord-import/badges/users.svg)](https://www.codetriage.com/zdennis/activerecord-import)
|
611
667
|
|
@@ -613,7 +669,7 @@ You can triage issues which may include reproducing bug reports or asking for vi
|
|
613
669
|
|
614
670
|
# License
|
615
671
|
|
616
|
-
This is licensed under the
|
672
|
+
This is licensed under the MIT license.
|
617
673
|
|
618
674
|
# Author
|
619
675
|
|
data/Rakefile
CHANGED
data/activerecord-import.gemspec
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require File.expand_path('../lib/activerecord-import/version', __FILE__)
|
3
4
|
|
4
5
|
Gem::Specification.new do |gem|
|
@@ -6,8 +7,12 @@ Gem::Specification.new do |gem|
|
|
6
7
|
gem.email = ["zach.dennis@gmail.com"]
|
7
8
|
gem.summary = "Bulk insert extension for ActiveRecord"
|
8
9
|
gem.description = "A library for bulk inserting data using ActiveRecord."
|
9
|
-
gem.homepage = "
|
10
|
-
gem.license = "
|
10
|
+
gem.homepage = "https://github.com/zdennis/activerecord-import"
|
11
|
+
gem.license = "MIT"
|
12
|
+
|
13
|
+
gem.metadata = {
|
14
|
+
"changelog_uri" => "https://github.com/zdennis/activerecord-import/blob/master/CHANGELOG.md"
|
15
|
+
}
|
11
16
|
|
12
17
|
gem.files = `git ls-files`.split($\)
|
13
18
|
gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
|
@@ -16,8 +21,8 @@ Gem::Specification.new do |gem|
|
|
16
21
|
gem.require_paths = ["lib"]
|
17
22
|
gem.version = ActiveRecord::Import::VERSION
|
18
23
|
|
19
|
-
gem.required_ruby_version = ">=
|
24
|
+
gem.required_ruby_version = ">= 2.4.0"
|
20
25
|
|
21
|
-
gem.add_runtime_dependency "activerecord", ">=
|
26
|
+
gem.add_runtime_dependency "activerecord", ">= 4.2"
|
22
27
|
gem.add_development_dependency "rake"
|
23
28
|
end
|
data/benchmarks/benchmark.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'pathname'
|
2
4
|
require "fileutils"
|
3
5
|
require "active_record"
|
@@ -20,7 +22,11 @@ FileUtils.mkdir_p 'log'
|
|
20
22
|
ActiveRecord::Base.configurations["test"] = YAML.load_file(File.join(benchmark_dir, "../test/database.yml"))[options.adapter]
|
21
23
|
ActiveRecord::Base.logger = Logger.new("log/test.log")
|
22
24
|
ActiveRecord::Base.logger.level = Logger::DEBUG
|
23
|
-
ActiveRecord
|
25
|
+
if ActiveRecord.respond_to?(:default_timezone)
|
26
|
+
ActiveRecord.default_timezone = :utc
|
27
|
+
else
|
28
|
+
ActiveRecord::Base.default_timezone = :utc
|
29
|
+
end
|
24
30
|
|
25
31
|
require "activerecord-import"
|
26
32
|
ActiveRecord::Base.establish_connection(:test)
|
@@ -35,11 +41,9 @@ require File.join(benchmark_dir, "../test/schema/generic_schema")
|
|
35
41
|
adapter_schema = File.join(benchmark_dir, "schema/#{options.adapter}_schema.rb")
|
36
42
|
require adapter_schema if File.exist?(adapter_schema)
|
37
43
|
|
38
|
-
Dir[File.dirname(__FILE__)
|
44
|
+
Dir["#{File.dirname(__FILE__)}/models/*.rb"].sort.each { |file| require file }
|
39
45
|
|
40
46
|
require File.join( benchmark_dir, 'lib', "#{options.adapter}_benchmark" )
|
41
|
-
|
42
|
-
table_types = nil
|
43
47
|
table_types = if options.benchmark_all_types
|
44
48
|
["all"]
|
45
49
|
else
|
@@ -47,8 +51,8 @@ else
|
|
47
51
|
end
|
48
52
|
|
49
53
|
letter = options.adapter[0].chr
|
50
|
-
clazz_str = letter.upcase + options.adapter[1
|
51
|
-
clazz = Object.const_get( clazz_str
|
54
|
+
clazz_str = letter.upcase + options.adapter[1..].downcase
|
55
|
+
clazz = Object.const_get( "#{clazz_str}Benchmark" )
|
52
56
|
|
53
57
|
benchmarks = []
|
54
58
|
options.number_of_objects.each do |num|
|
data/benchmarks/lib/base.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
class BenchmarkBase
|
2
4
|
attr_reader :results
|
3
5
|
|
@@ -14,7 +16,7 @@ class BenchmarkBase
|
|
14
16
|
end
|
15
17
|
end
|
16
18
|
|
17
|
-
# Returns
|
19
|
+
# Returns a struct which contains two attritues, +description+ and +tms+ after performing an
|
18
20
|
# actual benchmark.
|
19
21
|
#
|
20
22
|
# == PARAMETERS
|
@@ -22,18 +24,21 @@ class BenchmarkBase
|
|
22
24
|
# * blk - the block of code to benchmark
|
23
25
|
#
|
24
26
|
# == RETURNS
|
25
|
-
#
|
27
|
+
# A struct object with the following attributes:
|
26
28
|
# * description - the description of the benchmark ran
|
27
29
|
# * tms - a Benchmark::Tms containing the results of the benchmark
|
28
|
-
|
30
|
+
|
31
|
+
BmStruct = Struct.new( :description, :tms, :failed, keyword_init: true )
|
32
|
+
|
33
|
+
def bm( description, &block )
|
29
34
|
tms = nil
|
30
35
|
puts "Benchmarking #{description}"
|
31
36
|
|
32
|
-
Benchmark.bm { |x| tms = x.report
|
37
|
+
Benchmark.bm { |x| tms = x.report(&block) }
|
33
38
|
delete_all
|
34
39
|
failed = false
|
35
40
|
|
36
|
-
|
41
|
+
BmStruct.new( description: description, tms: tms, failed: failed )
|
37
42
|
end
|
38
43
|
|
39
44
|
# Given a model class (ie: Topic), and an array of columns and value sets
|
@@ -1,5 +1,6 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'optparse'
|
2
|
-
require 'ostruct'
|
3
4
|
|
4
5
|
#
|
5
6
|
# == PARAMETERS
|
@@ -27,7 +28,7 @@ module BenchmarkOptionParser
|
|
27
28
|
print_valid_table_types( options, prefix: " " )
|
28
29
|
end
|
29
30
|
|
30
|
-
# TODO IMPLEMENT THIS
|
31
|
+
# TODO: IMPLEMENT THIS
|
31
32
|
def self.print_valid_table_types( options, hsh = { prefix: '' } )
|
32
33
|
if !options.table_types.keys.empty?
|
33
34
|
options.table_types.keys.sort.each { |type| puts hsh[:prefix].to_s + type.to_s }
|
@@ -36,8 +37,11 @@ module BenchmarkOptionParser
|
|
36
37
|
end
|
37
38
|
end
|
38
39
|
|
40
|
+
OptionsStruct = Struct.new( :adapter, :table_types, :delete_on_finish, :number_of_objects, :outputs,
|
41
|
+
:benchmark_all_types, keyword_init: true )
|
42
|
+
OutputStruct = Struct.new( :format, :filename, keyword_init: true )
|
39
43
|
def self.parse( args )
|
40
|
-
options =
|
44
|
+
options = OptionsStruct.new(
|
41
45
|
adapter: 'mysql2',
|
42
46
|
table_types: {},
|
43
47
|
delete_on_finish: true,
|
@@ -79,12 +83,12 @@ module BenchmarkOptionParser
|
|
79
83
|
|
80
84
|
# print results in CSV format
|
81
85
|
opts.on( "--to-csv [String]", "Print results in a CSV file format" ) do |filename|
|
82
|
-
options.outputs <<
|
86
|
+
options.outputs << OutputStruct.new( format: 'csv', filename: filename)
|
83
87
|
end
|
84
88
|
|
85
89
|
# print results in HTML format
|
86
90
|
opts.on( "--to-html [String]", "Print results in HTML format" ) do |filename|
|
87
|
-
options.outputs <<
|
91
|
+
options.outputs << OutputStruct.new( format: 'html', filename: filename )
|
88
92
|
end
|
89
93
|
end # end opt.parse!
|
90
94
|
|
@@ -98,7 +102,7 @@ module BenchmarkOptionParser
|
|
98
102
|
end
|
99
103
|
|
100
104
|
options.number_of_objects = [1000] if options.number_of_objects.empty?
|
101
|
-
options.outputs = [
|
105
|
+
options.outputs = [OutputStruct.new( format: 'html', filename: 'benchmark.html')] if options.outputs.empty?
|
102
106
|
|
103
107
|
print_options( options )
|
104
108
|
|
data/benchmarks/lib/float.rb
CHANGED
@@ -1,7 +1,9 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'erb'
|
2
4
|
|
3
5
|
module OutputToHTML
|
4
|
-
TEMPLATE_HEADER = <<"EOT"
|
6
|
+
TEMPLATE_HEADER = <<"EOT"
|
5
7
|
<div>
|
6
8
|
All times are rounded to the nearest thousandth for display purposes. Speedups next to each time are computed
|
7
9
|
before any rounding occurs. Also, all speedup calculations are computed by comparing a given time against
|
@@ -9,7 +11,7 @@ module OutputToHTML
|
|
9
11
|
</div>
|
10
12
|
EOT
|
11
13
|
|
12
|
-
TEMPLATE = <<"EOT"
|
14
|
+
TEMPLATE = <<"EOT"
|
13
15
|
<style>
|
14
16
|
td#benchmarkTitle {
|
15
17
|
border: 1px solid black;
|
data/docker-compose.yml
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
version: "3.5"
|
2
|
+
|
3
|
+
services:
|
4
|
+
mysql:
|
5
|
+
platform: linux/x86_64
|
6
|
+
image: mysql:5.7
|
7
|
+
volumes:
|
8
|
+
- mysql-data:/var/lib/mysql
|
9
|
+
ports:
|
10
|
+
- "3306:3306"
|
11
|
+
|
12
|
+
postgresql:
|
13
|
+
image: postgres:latest
|
14
|
+
volumes:
|
15
|
+
- postgresql-data:/var/lib/postgresql/data
|
16
|
+
ports:
|
17
|
+
- "5432:5432"
|
18
|
+
|
19
|
+
app:
|
20
|
+
build:
|
21
|
+
context: .
|
22
|
+
environment:
|
23
|
+
DB_HOST: mysql
|
24
|
+
AR_VERSION: 7.0
|
25
|
+
volumes:
|
26
|
+
- .:/usr/src/app
|
27
|
+
depends_on:
|
28
|
+
- mysql
|
29
|
+
- postgresql
|
30
|
+
command: tail -f /dev/null
|
31
|
+
|
32
|
+
volumes:
|
33
|
+
mysql-data:
|
34
|
+
postgresql-data:
|
data/gemfiles/5.2.gemfile
CHANGED
data/gemfiles/6.0.gemfile
CHANGED
data/gemfiles/6.1.gemfile
CHANGED
@@ -1,6 +1,8 @@
|
|
1
|
-
|
2
|
-
require "activerecord-import/adapters/mysql_adapter"
|
1
|
+
# frozen_string_literal: true
|
3
2
|
|
4
|
-
|
5
|
-
|
3
|
+
require "active_record/connection_adapters/mysql2_adapter"
|
4
|
+
require "activerecord-import/adapters/mysql2_adapter"
|
5
|
+
|
6
|
+
class ActiveRecord::ConnectionAdapters::Mysql2Adapter
|
7
|
+
include ActiveRecord::Import::Mysql2Adapter
|
6
8
|
end
|