activerecord-import 1.0.4 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yaml +159 -0
  3. data/.gitignore +5 -0
  4. data/.rubocop.yml +76 -7
  5. data/.rubocop_todo.yml +10 -16
  6. data/Brewfile +3 -1
  7. data/CHANGELOG.md +143 -3
  8. data/Dockerfile +23 -0
  9. data/Gemfile +28 -24
  10. data/LICENSE +21 -56
  11. data/README.markdown +83 -27
  12. data/Rakefile +3 -0
  13. data/activerecord-import.gemspec +10 -5
  14. data/benchmarks/benchmark.rb +10 -6
  15. data/benchmarks/lib/base.rb +10 -5
  16. data/benchmarks/lib/cli_parser.rb +10 -6
  17. data/benchmarks/lib/float.rb +2 -0
  18. data/benchmarks/lib/mysql2_benchmark.rb +2 -0
  19. data/benchmarks/lib/output_to_csv.rb +2 -0
  20. data/benchmarks/lib/output_to_html.rb +4 -2
  21. data/benchmarks/models/test_innodb.rb +2 -0
  22. data/benchmarks/models/test_memory.rb +2 -0
  23. data/benchmarks/models/test_myisam.rb +2 -0
  24. data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
  25. data/docker-compose.yml +34 -0
  26. data/gemfiles/5.2.gemfile +2 -0
  27. data/gemfiles/6.0.gemfile +3 -0
  28. data/gemfiles/6.1.gemfile +4 -1
  29. data/gemfiles/7.0.gemfile +4 -0
  30. data/gemfiles/7.1.gemfile +3 -0
  31. data/gemfiles/7.2.gemfile +3 -0
  32. data/gemfiles/8.0.gemfile +3 -0
  33. data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
  34. data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
  35. data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
  36. data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +2 -0
  37. data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
  38. data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
  39. data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
  40. data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
  41. data/lib/activerecord-import/active_record/adapters/trilogy_adapter.rb +8 -0
  42. data/lib/activerecord-import/adapters/abstract_adapter.rb +9 -6
  43. data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
  44. data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
  45. data/lib/activerecord-import/adapters/mysql_adapter.rb +30 -21
  46. data/lib/activerecord-import/adapters/postgresql_adapter.rb +68 -48
  47. data/lib/activerecord-import/adapters/sqlite3_adapter.rb +37 -30
  48. data/lib/activerecord-import/adapters/trilogy_adapter.rb +7 -0
  49. data/lib/activerecord-import/base.rb +3 -1
  50. data/lib/activerecord-import/import.rb +160 -58
  51. data/lib/activerecord-import/synchronize.rb +3 -1
  52. data/lib/activerecord-import/value_sets_parser.rb +5 -0
  53. data/lib/activerecord-import/version.rb +3 -1
  54. data/lib/activerecord-import.rb +2 -1
  55. data/test/adapters/jdbcmysql.rb +2 -0
  56. data/test/adapters/jdbcpostgresql.rb +2 -0
  57. data/test/adapters/jdbcsqlite3.rb +2 -0
  58. data/test/adapters/makara_postgis.rb +2 -0
  59. data/test/adapters/mysql2.rb +2 -0
  60. data/test/adapters/mysql2_makara.rb +2 -0
  61. data/test/adapters/mysql2spatial.rb +2 -0
  62. data/test/adapters/postgis.rb +2 -0
  63. data/test/adapters/postgresql.rb +2 -0
  64. data/test/adapters/postgresql_makara.rb +2 -0
  65. data/test/adapters/seamless_database_pool.rb +2 -0
  66. data/test/adapters/spatialite.rb +2 -0
  67. data/test/adapters/sqlite3.rb +2 -0
  68. data/test/adapters/trilogy.rb +9 -0
  69. data/test/database.yml.sample +7 -0
  70. data/test/{travis → github}/database.yml +9 -3
  71. data/test/import_test.rb +108 -41
  72. data/test/jdbcmysql/import_test.rb +5 -3
  73. data/test/jdbcpostgresql/import_test.rb +4 -2
  74. data/test/jdbcsqlite3/import_test.rb +4 -2
  75. data/test/makara_postgis/import_test.rb +4 -2
  76. data/test/models/account.rb +2 -0
  77. data/test/models/alarm.rb +2 -0
  78. data/test/models/animal.rb +8 -0
  79. data/test/models/author.rb +9 -0
  80. data/test/models/bike_maker.rb +3 -0
  81. data/test/models/book.rb +12 -3
  82. data/test/models/car.rb +2 -0
  83. data/test/models/card.rb +5 -0
  84. data/test/models/chapter.rb +2 -0
  85. data/test/models/composite_book.rb +19 -0
  86. data/test/models/composite_chapter.rb +12 -0
  87. data/test/models/customer.rb +18 -0
  88. data/test/models/deck.rb +8 -0
  89. data/test/models/dictionary.rb +2 -0
  90. data/test/models/discount.rb +2 -0
  91. data/test/models/end_note.rb +2 -0
  92. data/test/models/group.rb +2 -0
  93. data/test/models/order.rb +17 -0
  94. data/test/models/playing_card.rb +4 -0
  95. data/test/models/promotion.rb +2 -0
  96. data/test/models/question.rb +2 -0
  97. data/test/models/rule.rb +2 -0
  98. data/test/models/tag.rb +9 -1
  99. data/test/models/tag_alias.rb +11 -0
  100. data/test/models/topic.rb +8 -0
  101. data/test/models/user.rb +2 -0
  102. data/test/models/user_token.rb +2 -0
  103. data/test/models/vendor.rb +2 -0
  104. data/test/models/widget.rb +12 -3
  105. data/test/mysql2/import_test.rb +5 -3
  106. data/test/mysql2_makara/import_test.rb +5 -3
  107. data/test/mysqlspatial2/import_test.rb +5 -3
  108. data/test/postgis/import_test.rb +4 -2
  109. data/test/postgresql/import_test.rb +4 -2
  110. data/test/schema/generic_schema.rb +37 -1
  111. data/test/schema/jdbcpostgresql_schema.rb +3 -1
  112. data/test/schema/mysql2_schema.rb +2 -0
  113. data/test/schema/postgis_schema.rb +3 -1
  114. data/test/schema/postgresql_schema.rb +38 -4
  115. data/test/schema/sqlite3_schema.rb +2 -0
  116. data/test/schema/version.rb +2 -0
  117. data/test/sqlite3/import_test.rb +4 -2
  118. data/test/support/active_support/test_case_extensions.rb +3 -5
  119. data/test/support/assertions.rb +2 -0
  120. data/test/support/factories.rb +2 -0
  121. data/test/support/generate.rb +4 -2
  122. data/test/support/mysql/import_examples.rb +7 -8
  123. data/test/support/postgresql/import_examples.rb +121 -53
  124. data/test/support/shared_examples/on_duplicate_key_ignore.rb +2 -0
  125. data/test/support/shared_examples/on_duplicate_key_update.rb +69 -10
  126. data/test/support/shared_examples/recursive_import.rb +137 -1
  127. data/test/support/sqlite3/import_examples.rb +2 -1
  128. data/test/synchronize_test.rb +2 -0
  129. data/test/test_helper.rb +38 -24
  130. data/test/trilogy/import_test.rb +7 -0
  131. data/test/value_sets_bytes_parser_test.rb +3 -1
  132. data/test/value_sets_records_parser_test.rb +3 -1
  133. metadata +46 -22
  134. data/.travis.yml +0 -74
  135. data/gemfiles/3.2.gemfile +0 -2
  136. data/gemfiles/4.0.gemfile +0 -2
  137. data/gemfiles/4.1.gemfile +0 -2
  138. data/gemfiles/4.2.gemfile +0 -2
  139. data/gemfiles/5.0.gemfile +0 -2
  140. data/gemfiles/5.1.gemfile +0 -2
  141. data/lib/activerecord-import/mysql2.rb +0 -7
  142. data/lib/activerecord-import/postgresql.rb +0 -7
  143. data/lib/activerecord-import/sqlite3.rb +0 -7
data/LICENSE CHANGED
@@ -1,56 +1,21 @@
1
- Ruby is copyrighted free software by Yukihiro Matsumoto <matz@netlab.jp>.
2
- You can redistribute it and/or modify it under either the terms of the
3
- 2-clause BSDL (see the file BSDL), or the conditions below:
4
-
5
- 1. You may make and give away verbatim copies of the source form of the
6
- software without restriction, provided that you duplicate all of the
7
- original copyright notices and associated disclaimers.
8
-
9
- 2. You may modify your copy of the software in any way, provided that
10
- you do at least ONE of the following:
11
-
12
- a) place your modifications in the Public Domain or otherwise
13
- make them Freely Available, such as by posting said
14
- modifications to Usenet or an equivalent medium, or by allowing
15
- the author to include your modifications in the software.
16
-
17
- b) use the modified software only within your corporation or
18
- organization.
19
-
20
- c) give non-standard binaries non-standard names, with
21
- instructions on where to get the original software distribution.
22
-
23
- d) make other distribution arrangements with the author.
24
-
25
- 3. You may distribute the software in object code or binary form,
26
- provided that you do at least ONE of the following:
27
-
28
- a) distribute the binaries and library files of the software,
29
- together with instructions (in the manual page or equivalent)
30
- on where to get the original distribution.
31
-
32
- b) accompany the distribution with the machine-readable source of
33
- the software.
34
-
35
- c) give non-standard binaries non-standard names, with
36
- instructions on where to get the original software distribution.
37
-
38
- d) make other distribution arrangements with the author.
39
-
40
- 4. You may modify and include the part of the software into any other
41
- software (possibly commercial). But some files in the distribution
42
- are not written by the author, so that they are not under these terms.
43
-
44
- For the list of those files and their copying conditions, see the
45
- file LEGAL.
46
-
47
- 5. The scripts and library files supplied as input to or produced as
48
- output from the software do not automatically fall under the
49
- copyright of the software, but belong to whomever generated them,
50
- and may be sold commercially, and may be aggregated with this
51
- software.
52
-
53
- 6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
54
- IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
55
- WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
56
- PURPOSE.
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2020 Zach Dennis <zach.dennis@gmail.com>
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.markdown CHANGED
@@ -1,4 +1,4 @@
1
- # Activerecord-Import [![Build Status](https://travis-ci.org/zdennis/activerecord-import.svg?branch=master)](https://travis-ci.org/zdennis/activerecord-import)
1
+ # Activerecord-Import ![Build Status](https://github.com/zdennis/activerecord-import/actions/workflows/test.yaml/badge.svg)
2
2
 
3
3
  Activerecord-Import is a library for bulk inserting data using ActiveRecord.
4
4
 
@@ -60,6 +60,8 @@ The gem provides the following high-level features:
60
60
 
61
61
  #### Introduction
62
62
 
63
+ This gem adds an `import` method (or `bulk_import`, for compatibility with gems like `elasticsearch-model`; see [Conflicts With Other Gems](#conflicts-with-other-gems)) to ActiveRecord classes.
64
+
63
65
  Without `activerecord-import`, you'd write something like this:
64
66
 
65
67
  ```ruby
@@ -229,9 +231,22 @@ columns = [ :title ]
229
231
  Book.import columns, books, batch_size: 2
230
232
  ```
231
233
 
234
+ If your import is particularly large or slow (possibly due to [callbacks](#callbacks)) whilst batch importing, you might want a way to report back on progress. This is supported by passing a callable as the `batch_progress` option. e.g:
235
+
236
+ ```ruby
237
+ my_proc = ->(rows_size, num_batches, current_batch_number, batch_duration_in_secs) {
238
+ # Using the arguments provided to the callable, you can
239
+ # send an email, post to a websocket,
240
+ # update slack, alert if import is taking too long, etc.
241
+ }
242
+
243
+ Book.import columns, books, batch_size: 2, batch_progress: my_proc
244
+ ```
245
+
232
246
  #### Recursive
233
247
 
234
- NOTE: This only works with PostgreSQL.
248
+ > **Note**
249
+ > This only works with PostgreSQL and ActiveRecord objects. This won't work with hashes or arrays as recursive inputs.
235
250
 
236
251
  Assume that Books <code>has_many</code> Reviews.
237
252
 
@@ -247,20 +262,22 @@ Book.import books, recursive: true
247
262
 
248
263
  ### Options
249
264
 
250
- Key | Options | Default | Description
251
- ----------------------- | --------------------- | ------------------ | -----------
252
- :validate | `true`/`false` | `true` | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
253
- :validate_uniqueness | `true`/`false` | `false` | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
254
- :validate_with_context | `Symbol` |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
255
- :on_duplicate_key_ignore| `true`/`false` | `false` | Allows skipping records with duplicate keys. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-ignore) for more details.
256
- :ignore | `true`/`false` | `false` | Alias for :on_duplicate_key_ignore.
257
- :on_duplicate_key_update| :all, `Array`, `Hash` | N/A | Allows upsert logic to be used. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-update) for more details.
258
- :synchronize | `Array` | N/A | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
259
- :timestamps | `true`/`false` | `true` | Enables/disables timestamps on imported records.
260
- :recursive | `true`/`false` | `false` | Imports has_many/has_one associations (PostgreSQL only).
261
- :batch_size | `Integer` | total # of records | Max number of records to insert per import
262
- :raise_error | `true`/`false` | `false` | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
263
- :all_or_none | `true`/`false` | `false` | Will not import any records if there is a record with validation errors.
265
+ Key | Options | Default | Description
266
+ ------------------------- | --------------------- | ------------------ | -----------
267
+ :validate | `true`/`false` | `true` | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
268
+ :validate_uniqueness | `true`/`false` | `false` | Whether or not to run ActiveRecord uniqueness validations. Beware this will incur an sql query per-record (N+1 queries). (requires `>= v0.27.0`).
269
+ :validate_with_context | `Symbol` |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
270
+ :track_validation_failures| `true`/`false` | `false` | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]`
271
+ :on_duplicate_key_ignore | `true`/`false` | `false` | Allows skipping records with duplicate keys. See [here](#duplicate-key-ignore) for more details.
272
+ :ignore | `true`/`false` | `false` | Alias for :on_duplicate_key_ignore.
273
+ :on_duplicate_key_update | :all, `Array`, `Hash` | N/A | Allows upsert logic to be used. See [here](#duplicate-key-update) for more details.
274
+ :synchronize | `Array` | N/A | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
275
+ :timestamps | `true`/`false` | `true` | Enables/disables timestamps on imported records.
276
+ :recursive | `true`/`false` | `false` | Imports has_many/has_one associations (PostgreSQL only).
277
+ :recursive_on_duplicate_key_update | `Hash` | N/A | Allows upsert logic to be used for recursive associations. The hash key is the association name and the value has the same options as `:on_duplicate_key_update`. See [here](#duplicate-key-update) for more details.
278
+ :batch_size | `Integer` | total # of records | Max number of records to insert per import
279
+ :raise_error | `true`/`false` | `false` | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
280
+ :all_or_none | `true`/`false` | `false` | Will not import any records if there is a record with validation errors.
264
281
 
265
282
  #### Duplicate Key Ignore
266
283
 
@@ -348,6 +365,29 @@ book.reload.title # => "Book1" (stayed the same)
348
365
  book.reload.author # => "Bob Barker" (changed)
349
366
  ```
350
367
 
368
+ PostgreSQL Using partial indexes
369
+
370
+ ```ruby
371
+ book = Book.create! title: "Book1", author: "George Orwell", published_at: Time.now
372
+ book.author = "Bob Barker"
373
+
374
+ # in migration
375
+ execute <<-SQL
376
+ CREATE INDEX books_published_at_index ON books (published_at) WHERE published_at IS NOT NULL;
377
+ SQL
378
+
379
+ # PostgreSQL version
380
+ Book.import [book], on_duplicate_key_update: {
381
+ conflict_target: [:id],
382
+ index_predicate: "published_at IS NOT NULL",
383
+ columns: [:author]
384
+ }
385
+
386
+ book.reload.title # => "Book1" (stayed the same)
387
+ book.reload.author # => "Bob Barker" (changed)
388
+ book.reload.published_at # => 2017-10-09 (stayed the same)
389
+ ```
390
+
351
391
  PostgreSQL Using constraints
352
392
 
353
393
  ```ruby
@@ -376,7 +416,7 @@ Book.import books, validate_uniqueness: true
376
416
 
377
417
  ### Return Info
378
418
 
379
- The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed`.
419
+ The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed.
380
420
 
381
421
  ```ruby
382
422
  articles = [
@@ -417,7 +457,9 @@ Should you wish to specify those columns, you may use the option `timestamps: fa
417
457
 
418
458
  However, it is also possible to set just `:created_at` in specific records. In this case despite using `timestamps: true`, `:created_at` will be updated only in records where that field is `nil`. Same rule applies for record associations when enabling the option `recursive: true`.
419
459
 
420
- If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is
460
+ If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is.
461
+ > **Note**
462
+ > If you are using ActiveRecord 7.0 or later, please use `ActiveRecord.default_timezone` instead.
421
463
 
422
464
  ### Callbacks
423
465
 
@@ -489,7 +531,8 @@ This allows an external gem to dynamically add an adapter without the need to ad
489
531
 
490
532
  ### Requiring
491
533
 
492
- Note: These instructions will only work if you are using version 0.2.0 or higher.
534
+ > **Note**
535
+ > These instructions will only work if you are using version 0.2.0 or higher.
493
536
 
494
537
  #### Autoloading via Bundler
495
538
 
@@ -526,11 +569,11 @@ require 'activerecord-import'
526
569
  ### Load Path Setup
527
570
  To understand how rubygems loads code you can reference the following:
528
571
 
529
- http://guides.rubygems.org/patterns/#loading_code
572
+ https://guides.rubygems.org/patterns/#loading-code
530
573
 
531
574
  And an example of how active_record dynamically load adapters:
532
575
 
533
- https://github.com/rails/rails/blob/master/activerecord/lib/active_record/connection_adapters/connection_specification.rb
576
+ https://github.com/rails/rails/blob/main/activerecord/lib/active_record/connection_adapters.rb
534
577
 
535
578
  In summary, when a gem is loaded rubygems adds the `lib` folder of the gem to the global load path `$LOAD_PATH` so that all `require` lookups will not propagate through all of the folders on the load path. When a `require` is issued each folder on the `$LOAD_PATH` is checked for the file and/or folder referenced. This allows a gem (like activerecord-import) to define push the activerecord-import folder (or namespace) on the `$LOAD_PATH` and any adapters provided by activerecord-import will be found by rubygems when the require is issued.
536
579
 
@@ -556,7 +599,7 @@ When rubygems pushes the `lib` folder onto the load path a `require` will now fi
556
599
 
557
600
  Activerecord-Import adds the `.import` method onto `ActiveRecord::Base`. There are other gems, such as `elasticsearch-rails`, that do the same thing. In conflicts such as this, there is an aliased method named `.bulk_import` that can be used interchangeably.
558
601
 
559
- If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by explcitly setting this value within the model. For example:
602
+ If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by explicitly setting this value within the model. For example:
560
603
 
561
604
  ```ruby
562
605
  class Post < ActiveRecord::Base
@@ -599,13 +642,26 @@ After that, you can run the tests. They run against multiple tests and ActiveRec
599
642
 
600
643
  This is one example of how to run the tests:
601
644
 
602
- ```ruby
645
+ ```bash
603
646
  rm Gemfile.lock
604
- AR_VERSION=4.2 bundle install
605
- AR_VERSION=4.2 bundle exec rake test:postgresql test:sqlite3 test:mysql2
647
+ AR_VERSION=7.0 bundle install
648
+ AR_VERSION=7.0 bundle exec rake test:postgresql test:sqlite3 test:mysql2
649
+ ```
650
+
651
+ Once you have pushed up your changes, you can find your CI results [here](https://github.com/zdennis/activerecord-import/actions).
652
+
653
+ #### Docker Setup
654
+
655
+ Before you begin, make sure you have [Docker](https://www.docker.com/products/docker-desktop/) and [Docker Compose](https://docs.docker.com/compose/) installed on your machine. If you don't, you can install both via Homebrew using the following command:
656
+
657
+ ```bash
658
+ brew install docker && brew install docker-compose
606
659
  ```
660
+ ##### Steps
607
661
 
608
- Once you have pushed up your changes, you can find your CI results [here](https://travis-ci.org/zdennis/activerecord-import/).
662
+ 1. In your terminal run `docker-compose up --build`
663
+ 1. In another tab/window run `docker-compose exec app bash`
664
+ 1. In that same terminal run the mysql2 test by running `bundle exec rake test:mysql2`
609
665
 
610
666
  ## Issue Triage [![Open Source Helpers](https://www.codetriage.com/zdennis/activerecord-import/badges/users.svg)](https://www.codetriage.com/zdennis/activerecord-import)
611
667
 
@@ -613,7 +669,7 @@ You can triage issues which may include reproducing bug reports or asking for vi
613
669
 
614
670
  # License
615
671
 
616
- This is licensed under the ruby license.
672
+ This is licensed under the MIT license.
617
673
 
618
674
  # Author
619
675
 
data/Rakefile CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "bundler"
2
4
  Bundler.setup
3
5
 
@@ -27,6 +29,7 @@ ADAPTERS = %w(
27
29
  sqlite3
28
30
  spatialite
29
31
  seamless_database_pool
32
+ trilogy
30
33
  ).freeze
31
34
  ADAPTERS.each do |adapter|
32
35
  namespace :test do
@@ -1,4 +1,5 @@
1
- # -*- encoding: utf-8 -*-
1
+ # frozen_string_literal: true
2
+
2
3
  require File.expand_path('../lib/activerecord-import/version', __FILE__)
3
4
 
4
5
  Gem::Specification.new do |gem|
@@ -6,8 +7,12 @@ Gem::Specification.new do |gem|
6
7
  gem.email = ["zach.dennis@gmail.com"]
7
8
  gem.summary = "Bulk insert extension for ActiveRecord"
8
9
  gem.description = "A library for bulk inserting data using ActiveRecord."
9
- gem.homepage = "http://github.com/zdennis/activerecord-import"
10
- gem.license = "Ruby"
10
+ gem.homepage = "https://github.com/zdennis/activerecord-import"
11
+ gem.license = "MIT"
12
+
13
+ gem.metadata = {
14
+ "changelog_uri" => "https://github.com/zdennis/activerecord-import/blob/master/CHANGELOG.md"
15
+ }
11
16
 
12
17
  gem.files = `git ls-files`.split($\)
13
18
  gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
@@ -16,8 +21,8 @@ Gem::Specification.new do |gem|
16
21
  gem.require_paths = ["lib"]
17
22
  gem.version = ActiveRecord::Import::VERSION
18
23
 
19
- gem.required_ruby_version = ">= 1.9.2"
24
+ gem.required_ruby_version = ">= 2.4.0"
20
25
 
21
- gem.add_runtime_dependency "activerecord", ">= 3.2"
26
+ gem.add_runtime_dependency "activerecord", ">= 4.2"
22
27
  gem.add_development_dependency "rake"
23
28
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'pathname'
2
4
  require "fileutils"
3
5
  require "active_record"
@@ -20,7 +22,11 @@ FileUtils.mkdir_p 'log'
20
22
  ActiveRecord::Base.configurations["test"] = YAML.load_file(File.join(benchmark_dir, "../test/database.yml"))[options.adapter]
21
23
  ActiveRecord::Base.logger = Logger.new("log/test.log")
22
24
  ActiveRecord::Base.logger.level = Logger::DEBUG
23
- ActiveRecord::Base.default_timezone = :utc
25
+ if ActiveRecord.respond_to?(:default_timezone)
26
+ ActiveRecord.default_timezone = :utc
27
+ else
28
+ ActiveRecord::Base.default_timezone = :utc
29
+ end
24
30
 
25
31
  require "activerecord-import"
26
32
  ActiveRecord::Base.establish_connection(:test)
@@ -35,11 +41,9 @@ require File.join(benchmark_dir, "../test/schema/generic_schema")
35
41
  adapter_schema = File.join(benchmark_dir, "schema/#{options.adapter}_schema.rb")
36
42
  require adapter_schema if File.exist?(adapter_schema)
37
43
 
38
- Dir[File.dirname(__FILE__) + "/models/*.rb"].each { |file| require file }
44
+ Dir["#{File.dirname(__FILE__)}/models/*.rb"].sort.each { |file| require file }
39
45
 
40
46
  require File.join( benchmark_dir, 'lib', "#{options.adapter}_benchmark" )
41
-
42
- table_types = nil
43
47
  table_types = if options.benchmark_all_types
44
48
  ["all"]
45
49
  else
@@ -47,8 +51,8 @@ else
47
51
  end
48
52
 
49
53
  letter = options.adapter[0].chr
50
- clazz_str = letter.upcase + options.adapter[1..-1].downcase
51
- clazz = Object.const_get( clazz_str + "Benchmark" )
54
+ clazz_str = letter.upcase + options.adapter[1..].downcase
55
+ clazz = Object.const_get( "#{clazz_str}Benchmark" )
52
56
 
53
57
  benchmarks = []
54
58
  options.number_of_objects.each do |num|
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  class BenchmarkBase
2
4
  attr_reader :results
3
5
 
@@ -14,7 +16,7 @@ class BenchmarkBase
14
16
  end
15
17
  end
16
18
 
17
- # Returns an OpenStruct which contains two attritues, +description+ and +tms+ after performing an
19
+ # Returns a struct which contains two attritues, +description+ and +tms+ after performing an
18
20
  # actual benchmark.
19
21
  #
20
22
  # == PARAMETERS
@@ -22,18 +24,21 @@ class BenchmarkBase
22
24
  # * blk - the block of code to benchmark
23
25
  #
24
26
  # == RETURNS
25
- # An OpenStruct object with the following attributes:
27
+ # A struct object with the following attributes:
26
28
  # * description - the description of the benchmark ran
27
29
  # * tms - a Benchmark::Tms containing the results of the benchmark
28
- def bm( description )
30
+
31
+ BmStruct = Struct.new( :description, :tms, :failed, keyword_init: true )
32
+
33
+ def bm( description, &block )
29
34
  tms = nil
30
35
  puts "Benchmarking #{description}"
31
36
 
32
- Benchmark.bm { |x| tms = x.report { yield } }
37
+ Benchmark.bm { |x| tms = x.report(&block) }
33
38
  delete_all
34
39
  failed = false
35
40
 
36
- OpenStruct.new description: description, tms: tms, failed: failed
41
+ BmStruct.new( description: description, tms: tms, failed: failed )
37
42
  end
38
43
 
39
44
  # Given a model class (ie: Topic), and an array of columns and value sets
@@ -1,5 +1,6 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'optparse'
2
- require 'ostruct'
3
4
 
4
5
  #
5
6
  # == PARAMETERS
@@ -27,7 +28,7 @@ module BenchmarkOptionParser
27
28
  print_valid_table_types( options, prefix: " " )
28
29
  end
29
30
 
30
- # TODO IMPLEMENT THIS
31
+ # TODO: IMPLEMENT THIS
31
32
  def self.print_valid_table_types( options, hsh = { prefix: '' } )
32
33
  if !options.table_types.keys.empty?
33
34
  options.table_types.keys.sort.each { |type| puts hsh[:prefix].to_s + type.to_s }
@@ -36,8 +37,11 @@ module BenchmarkOptionParser
36
37
  end
37
38
  end
38
39
 
40
+ OptionsStruct = Struct.new( :adapter, :table_types, :delete_on_finish, :number_of_objects, :outputs,
41
+ :benchmark_all_types, keyword_init: true )
42
+ OutputStruct = Struct.new( :format, :filename, keyword_init: true )
39
43
  def self.parse( args )
40
- options = OpenStruct.new(
44
+ options = OptionsStruct.new(
41
45
  adapter: 'mysql2',
42
46
  table_types: {},
43
47
  delete_on_finish: true,
@@ -79,12 +83,12 @@ module BenchmarkOptionParser
79
83
 
80
84
  # print results in CSV format
81
85
  opts.on( "--to-csv [String]", "Print results in a CSV file format" ) do |filename|
82
- options.outputs << OpenStruct.new( format: 'csv', filename: filename)
86
+ options.outputs << OutputStruct.new( format: 'csv', filename: filename)
83
87
  end
84
88
 
85
89
  # print results in HTML format
86
90
  opts.on( "--to-html [String]", "Print results in HTML format" ) do |filename|
87
- options.outputs << OpenStruct.new( format: 'html', filename: filename )
91
+ options.outputs << OutputStruct.new( format: 'html', filename: filename )
88
92
  end
89
93
  end # end opt.parse!
90
94
 
@@ -98,7 +102,7 @@ module BenchmarkOptionParser
98
102
  end
99
103
 
100
104
  options.number_of_objects = [1000] if options.number_of_objects.empty?
101
- options.outputs = [OpenStruct.new( format: 'html', filename: 'benchmark.html')] if options.outputs.empty?
105
+ options.outputs = [OutputStruct.new( format: 'html', filename: 'benchmark.html')] if options.outputs.empty?
102
106
 
103
107
  print_options( options )
104
108
 
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  # Taken from http://www.programmingishard.com/posts/show/128
2
4
  # Posted by rbates
3
5
  class Float
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  class Mysql2Benchmark < BenchmarkBase
2
4
  def benchmark_all( array_of_cols_and_vals )
3
5
  methods = self.methods.find_all { |m| m =~ /benchmark_/ }
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'csv'
2
4
 
3
5
  module OutputToCSV
@@ -1,7 +1,9 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'erb'
2
4
 
3
5
  module OutputToHTML
4
- TEMPLATE_HEADER = <<"EOT".freeze
6
+ TEMPLATE_HEADER = <<"EOT"
5
7
  <div>
6
8
  All times are rounded to the nearest thousandth for display purposes. Speedups next to each time are computed
7
9
  before any rounding occurs. Also, all speedup calculations are computed by comparing a given time against
@@ -9,7 +11,7 @@ module OutputToHTML
9
11
  </div>
10
12
  EOT
11
13
 
12
- TEMPLATE = <<"EOT".freeze
14
+ TEMPLATE = <<"EOT"
13
15
  <style>
14
16
  td#benchmarkTitle {
15
17
  border: 1px solid black;
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  class TestInnoDb < ActiveRecord::Base
2
4
  self.table_name = 'test_innodb'
3
5
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  class TestMemory < ActiveRecord::Base
2
4
  self.table_name = 'test_memory'
3
5
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  class TestMyISAM < ActiveRecord::Base
2
4
  self.table_name = 'test_myisam'
3
5
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  ActiveRecord::Schema.define do
2
4
  create_table :test_myisam, options: 'ENGINE=MyISAM', force: true do |t|
3
5
  t.column :my_name, :string, null: false
@@ -0,0 +1,34 @@
1
+ version: "3.5"
2
+
3
+ services:
4
+ mysql:
5
+ platform: linux/x86_64
6
+ image: mysql:5.7
7
+ volumes:
8
+ - mysql-data:/var/lib/mysql
9
+ ports:
10
+ - "3306:3306"
11
+
12
+ postgresql:
13
+ image: postgres:latest
14
+ volumes:
15
+ - postgresql-data:/var/lib/postgresql/data
16
+ ports:
17
+ - "5432:5432"
18
+
19
+ app:
20
+ build:
21
+ context: .
22
+ environment:
23
+ DB_HOST: mysql
24
+ AR_VERSION: 7.0
25
+ volumes:
26
+ - .:/usr/src/app
27
+ depends_on:
28
+ - mysql
29
+ - postgresql
30
+ command: tail -f /dev/null
31
+
32
+ volumes:
33
+ mysql-data:
34
+ postgresql-data:
data/gemfiles/5.2.gemfile CHANGED
@@ -1,2 +1,4 @@
1
+ # frozen_string_literal: true
2
+
1
3
  gem 'activerecord', '~> 5.2.0'
2
4
  gem 'composite_primary_keys', '~> 11.0'
data/gemfiles/6.0.gemfile CHANGED
@@ -1 +1,4 @@
1
+ # frozen_string_literal: true
2
+
1
3
  gem 'activerecord', '~> 6.0.0'
4
+ gem 'composite_primary_keys', '~> 12.0'
data/gemfiles/6.1.gemfile CHANGED
@@ -1 +1,4 @@
1
- gem 'activerecord', '~> 6.1.0.alpha', github: "rails/rails"
1
+ # frozen_string_literal: true
2
+
3
+ gem 'activerecord', '~> 6.1.0'
4
+ gem 'composite_primary_keys', '~> 13.0'
@@ -0,0 +1,4 @@
1
+ # frozen_string_literal: true
2
+
3
+ gem 'activerecord', '~> 7.0.0'
4
+ gem 'composite_primary_keys', '~> 14.0'
@@ -0,0 +1,3 @@
1
+ # frozen_string_literal: true
2
+
3
+ gem 'activerecord', '~> 7.1.0'
@@ -0,0 +1,3 @@
1
+ # frozen_string_literal: true
2
+
3
+ gem 'activerecord', '~> 7.2.0'
@@ -0,0 +1,3 @@
1
+ # frozen_string_literal: true
2
+
3
+ gem 'activerecord', '~> 8.0.0'
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "activerecord-import/adapters/abstract_adapter"
2
4
 
3
5
  module ActiveRecord # :nodoc:
@@ -1,6 +1,8 @@
1
- require "active_record/connection_adapters/mysql_adapter"
2
- require "activerecord-import/adapters/mysql_adapter"
1
+ # frozen_string_literal: true
3
2
 
4
- class ActiveRecord::ConnectionAdapters::MysqlAdapter
5
- include ActiveRecord::Import::MysqlAdapter
3
+ require "active_record/connection_adapters/mysql2_adapter"
4
+ require "activerecord-import/adapters/mysql2_adapter"
5
+
6
+ class ActiveRecord::ConnectionAdapters::Mysql2Adapter
7
+ include ActiveRecord::Import::Mysql2Adapter
6
8
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "active_record/connection_adapters/postgresql_adapter"
2
4
  require "activerecord-import/adapters/postgresql_adapter"
3
5
 
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "active_record/connection_adapters/sqlite3_adapter"
2
4
  require "activerecord-import/adapters/sqlite3_adapter"
3
5