activerecord-import 1.0.4 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (143) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yaml +159 -0
  3. data/.gitignore +5 -0
  4. data/.rubocop.yml +76 -7
  5. data/.rubocop_todo.yml +10 -16
  6. data/Brewfile +3 -1
  7. data/CHANGELOG.md +143 -3
  8. data/Dockerfile +23 -0
  9. data/Gemfile +28 -24
  10. data/LICENSE +21 -56
  11. data/README.markdown +83 -27
  12. data/Rakefile +3 -0
  13. data/activerecord-import.gemspec +10 -5
  14. data/benchmarks/benchmark.rb +10 -6
  15. data/benchmarks/lib/base.rb +10 -5
  16. data/benchmarks/lib/cli_parser.rb +10 -6
  17. data/benchmarks/lib/float.rb +2 -0
  18. data/benchmarks/lib/mysql2_benchmark.rb +2 -0
  19. data/benchmarks/lib/output_to_csv.rb +2 -0
  20. data/benchmarks/lib/output_to_html.rb +4 -2
  21. data/benchmarks/models/test_innodb.rb +2 -0
  22. data/benchmarks/models/test_memory.rb +2 -0
  23. data/benchmarks/models/test_myisam.rb +2 -0
  24. data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
  25. data/docker-compose.yml +34 -0
  26. data/gemfiles/5.2.gemfile +2 -0
  27. data/gemfiles/6.0.gemfile +3 -0
  28. data/gemfiles/6.1.gemfile +4 -1
  29. data/gemfiles/7.0.gemfile +4 -0
  30. data/gemfiles/7.1.gemfile +3 -0
  31. data/gemfiles/7.2.gemfile +3 -0
  32. data/gemfiles/8.0.gemfile +3 -0
  33. data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
  34. data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
  35. data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
  36. data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +2 -0
  37. data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
  38. data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
  39. data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
  40. data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
  41. data/lib/activerecord-import/active_record/adapters/trilogy_adapter.rb +8 -0
  42. data/lib/activerecord-import/adapters/abstract_adapter.rb +9 -6
  43. data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
  44. data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
  45. data/lib/activerecord-import/adapters/mysql_adapter.rb +30 -21
  46. data/lib/activerecord-import/adapters/postgresql_adapter.rb +68 -48
  47. data/lib/activerecord-import/adapters/sqlite3_adapter.rb +37 -30
  48. data/lib/activerecord-import/adapters/trilogy_adapter.rb +7 -0
  49. data/lib/activerecord-import/base.rb +3 -1
  50. data/lib/activerecord-import/import.rb +160 -58
  51. data/lib/activerecord-import/synchronize.rb +3 -1
  52. data/lib/activerecord-import/value_sets_parser.rb +5 -0
  53. data/lib/activerecord-import/version.rb +3 -1
  54. data/lib/activerecord-import.rb +2 -1
  55. data/test/adapters/jdbcmysql.rb +2 -0
  56. data/test/adapters/jdbcpostgresql.rb +2 -0
  57. data/test/adapters/jdbcsqlite3.rb +2 -0
  58. data/test/adapters/makara_postgis.rb +2 -0
  59. data/test/adapters/mysql2.rb +2 -0
  60. data/test/adapters/mysql2_makara.rb +2 -0
  61. data/test/adapters/mysql2spatial.rb +2 -0
  62. data/test/adapters/postgis.rb +2 -0
  63. data/test/adapters/postgresql.rb +2 -0
  64. data/test/adapters/postgresql_makara.rb +2 -0
  65. data/test/adapters/seamless_database_pool.rb +2 -0
  66. data/test/adapters/spatialite.rb +2 -0
  67. data/test/adapters/sqlite3.rb +2 -0
  68. data/test/adapters/trilogy.rb +9 -0
  69. data/test/database.yml.sample +7 -0
  70. data/test/{travis → github}/database.yml +9 -3
  71. data/test/import_test.rb +108 -41
  72. data/test/jdbcmysql/import_test.rb +5 -3
  73. data/test/jdbcpostgresql/import_test.rb +4 -2
  74. data/test/jdbcsqlite3/import_test.rb +4 -2
  75. data/test/makara_postgis/import_test.rb +4 -2
  76. data/test/models/account.rb +2 -0
  77. data/test/models/alarm.rb +2 -0
  78. data/test/models/animal.rb +8 -0
  79. data/test/models/author.rb +9 -0
  80. data/test/models/bike_maker.rb +3 -0
  81. data/test/models/book.rb +12 -3
  82. data/test/models/car.rb +2 -0
  83. data/test/models/card.rb +5 -0
  84. data/test/models/chapter.rb +2 -0
  85. data/test/models/composite_book.rb +19 -0
  86. data/test/models/composite_chapter.rb +12 -0
  87. data/test/models/customer.rb +18 -0
  88. data/test/models/deck.rb +8 -0
  89. data/test/models/dictionary.rb +2 -0
  90. data/test/models/discount.rb +2 -0
  91. data/test/models/end_note.rb +2 -0
  92. data/test/models/group.rb +2 -0
  93. data/test/models/order.rb +17 -0
  94. data/test/models/playing_card.rb +4 -0
  95. data/test/models/promotion.rb +2 -0
  96. data/test/models/question.rb +2 -0
  97. data/test/models/rule.rb +2 -0
  98. data/test/models/tag.rb +9 -1
  99. data/test/models/tag_alias.rb +11 -0
  100. data/test/models/topic.rb +8 -0
  101. data/test/models/user.rb +2 -0
  102. data/test/models/user_token.rb +2 -0
  103. data/test/models/vendor.rb +2 -0
  104. data/test/models/widget.rb +12 -3
  105. data/test/mysql2/import_test.rb +5 -3
  106. data/test/mysql2_makara/import_test.rb +5 -3
  107. data/test/mysqlspatial2/import_test.rb +5 -3
  108. data/test/postgis/import_test.rb +4 -2
  109. data/test/postgresql/import_test.rb +4 -2
  110. data/test/schema/generic_schema.rb +37 -1
  111. data/test/schema/jdbcpostgresql_schema.rb +3 -1
  112. data/test/schema/mysql2_schema.rb +2 -0
  113. data/test/schema/postgis_schema.rb +3 -1
  114. data/test/schema/postgresql_schema.rb +38 -4
  115. data/test/schema/sqlite3_schema.rb +2 -0
  116. data/test/schema/version.rb +2 -0
  117. data/test/sqlite3/import_test.rb +4 -2
  118. data/test/support/active_support/test_case_extensions.rb +3 -5
  119. data/test/support/assertions.rb +2 -0
  120. data/test/support/factories.rb +2 -0
  121. data/test/support/generate.rb +4 -2
  122. data/test/support/mysql/import_examples.rb +7 -8
  123. data/test/support/postgresql/import_examples.rb +121 -53
  124. data/test/support/shared_examples/on_duplicate_key_ignore.rb +2 -0
  125. data/test/support/shared_examples/on_duplicate_key_update.rb +69 -10
  126. data/test/support/shared_examples/recursive_import.rb +137 -1
  127. data/test/support/sqlite3/import_examples.rb +2 -1
  128. data/test/synchronize_test.rb +2 -0
  129. data/test/test_helper.rb +38 -24
  130. data/test/trilogy/import_test.rb +7 -0
  131. data/test/value_sets_bytes_parser_test.rb +3 -1
  132. data/test/value_sets_records_parser_test.rb +3 -1
  133. metadata +46 -22
  134. data/.travis.yml +0 -74
  135. data/gemfiles/3.2.gemfile +0 -2
  136. data/gemfiles/4.0.gemfile +0 -2
  137. data/gemfiles/4.1.gemfile +0 -2
  138. data/gemfiles/4.2.gemfile +0 -2
  139. data/gemfiles/5.0.gemfile +0 -2
  140. data/gemfiles/5.1.gemfile +0 -2
  141. data/lib/activerecord-import/mysql2.rb +0 -7
  142. data/lib/activerecord-import/postgresql.rb +0 -7
  143. data/lib/activerecord-import/sqlite3.rb +0 -7
data/LICENSE CHANGED
@@ -1,56 +1,21 @@
1
- Ruby is copyrighted free software by Yukihiro Matsumoto <matz@netlab.jp>.
2
- You can redistribute it and/or modify it under either the terms of the
3
- 2-clause BSDL (see the file BSDL), or the conditions below:
4
-
5
- 1. You may make and give away verbatim copies of the source form of the
6
- software without restriction, provided that you duplicate all of the
7
- original copyright notices and associated disclaimers.
8
-
9
- 2. You may modify your copy of the software in any way, provided that
10
- you do at least ONE of the following:
11
-
12
- a) place your modifications in the Public Domain or otherwise
13
- make them Freely Available, such as by posting said
14
- modifications to Usenet or an equivalent medium, or by allowing
15
- the author to include your modifications in the software.
16
-
17
- b) use the modified software only within your corporation or
18
- organization.
19
-
20
- c) give non-standard binaries non-standard names, with
21
- instructions on where to get the original software distribution.
22
-
23
- d) make other distribution arrangements with the author.
24
-
25
- 3. You may distribute the software in object code or binary form,
26
- provided that you do at least ONE of the following:
27
-
28
- a) distribute the binaries and library files of the software,
29
- together with instructions (in the manual page or equivalent)
30
- on where to get the original distribution.
31
-
32
- b) accompany the distribution with the machine-readable source of
33
- the software.
34
-
35
- c) give non-standard binaries non-standard names, with
36
- instructions on where to get the original software distribution.
37
-
38
- d) make other distribution arrangements with the author.
39
-
40
- 4. You may modify and include the part of the software into any other
41
- software (possibly commercial). But some files in the distribution
42
- are not written by the author, so that they are not under these terms.
43
-
44
- For the list of those files and their copying conditions, see the
45
- file LEGAL.
46
-
47
- 5. The scripts and library files supplied as input to or produced as
48
- output from the software do not automatically fall under the
49
- copyright of the software, but belong to whomever generated them,
50
- and may be sold commercially, and may be aggregated with this
51
- software.
52
-
53
- 6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
54
- IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
55
- WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
56
- PURPOSE.
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2020 Zach Dennis <zach.dennis@gmail.com>
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.markdown CHANGED
@@ -1,4 +1,4 @@
1
- # Activerecord-Import [![Build Status](https://travis-ci.org/zdennis/activerecord-import.svg?branch=master)](https://travis-ci.org/zdennis/activerecord-import)
1
+ # Activerecord-Import ![Build Status](https://github.com/zdennis/activerecord-import/actions/workflows/test.yaml/badge.svg)
2
2
 
3
3
  Activerecord-Import is a library for bulk inserting data using ActiveRecord.
4
4
 
@@ -60,6 +60,8 @@ The gem provides the following high-level features:
60
60
 
61
61
  #### Introduction
62
62
 
63
+ This gem adds an `import` method (or `bulk_import`, for compatibility with gems like `elasticsearch-model`; see [Conflicts With Other Gems](#conflicts-with-other-gems)) to ActiveRecord classes.
64
+
63
65
  Without `activerecord-import`, you'd write something like this:
64
66
 
65
67
  ```ruby
@@ -229,9 +231,22 @@ columns = [ :title ]
229
231
  Book.import columns, books, batch_size: 2
230
232
  ```
231
233
 
234
+ If your import is particularly large or slow (possibly due to [callbacks](#callbacks)) whilst batch importing, you might want a way to report back on progress. This is supported by passing a callable as the `batch_progress` option. e.g:
235
+
236
+ ```ruby
237
+ my_proc = ->(rows_size, num_batches, current_batch_number, batch_duration_in_secs) {
238
+ # Using the arguments provided to the callable, you can
239
+ # send an email, post to a websocket,
240
+ # update slack, alert if import is taking too long, etc.
241
+ }
242
+
243
+ Book.import columns, books, batch_size: 2, batch_progress: my_proc
244
+ ```
245
+
232
246
  #### Recursive
233
247
 
234
- NOTE: This only works with PostgreSQL.
248
+ > **Note**
249
+ > This only works with PostgreSQL and ActiveRecord objects. This won't work with hashes or arrays as recursive inputs.
235
250
 
236
251
  Assume that Books <code>has_many</code> Reviews.
237
252
 
@@ -247,20 +262,22 @@ Book.import books, recursive: true
247
262
 
248
263
  ### Options
249
264
 
250
- Key | Options | Default | Description
251
- ----------------------- | --------------------- | ------------------ | -----------
252
- :validate | `true`/`false` | `true` | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
253
- :validate_uniqueness | `true`/`false` | `false` | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
254
- :validate_with_context | `Symbol` |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
255
- :on_duplicate_key_ignore| `true`/`false` | `false` | Allows skipping records with duplicate keys. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-ignore) for more details.
256
- :ignore | `true`/`false` | `false` | Alias for :on_duplicate_key_ignore.
257
- :on_duplicate_key_update| :all, `Array`, `Hash` | N/A | Allows upsert logic to be used. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-update) for more details.
258
- :synchronize | `Array` | N/A | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
259
- :timestamps | `true`/`false` | `true` | Enables/disables timestamps on imported records.
260
- :recursive | `true`/`false` | `false` | Imports has_many/has_one associations (PostgreSQL only).
261
- :batch_size | `Integer` | total # of records | Max number of records to insert per import
262
- :raise_error | `true`/`false` | `false` | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
263
- :all_or_none | `true`/`false` | `false` | Will not import any records if there is a record with validation errors.
265
+ Key | Options | Default | Description
266
+ ------------------------- | --------------------- | ------------------ | -----------
267
+ :validate | `true`/`false` | `true` | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
268
+ :validate_uniqueness | `true`/`false` | `false` | Whether or not to run ActiveRecord uniqueness validations. Beware this will incur an sql query per-record (N+1 queries). (requires `>= v0.27.0`).
269
+ :validate_with_context | `Symbol` |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
270
+ :track_validation_failures| `true`/`false` | `false` | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]`
271
+ :on_duplicate_key_ignore | `true`/`false` | `false` | Allows skipping records with duplicate keys. See [here](#duplicate-key-ignore) for more details.
272
+ :ignore | `true`/`false` | `false` | Alias for :on_duplicate_key_ignore.
273
+ :on_duplicate_key_update | :all, `Array`, `Hash` | N/A | Allows upsert logic to be used. See [here](#duplicate-key-update) for more details.
274
+ :synchronize | `Array` | N/A | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
275
+ :timestamps | `true`/`false` | `true` | Enables/disables timestamps on imported records.
276
+ :recursive | `true`/`false` | `false` | Imports has_many/has_one associations (PostgreSQL only).
277
+ :recursive_on_duplicate_key_update | `Hash` | N/A | Allows upsert logic to be used for recursive associations. The hash key is the association name and the value has the same options as `:on_duplicate_key_update`. See [here](#duplicate-key-update) for more details.
278
+ :batch_size | `Integer` | total # of records | Max number of records to insert per import
279
+ :raise_error | `true`/`false` | `false` | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
280
+ :all_or_none | `true`/`false` | `false` | Will not import any records if there is a record with validation errors.
264
281
 
265
282
  #### Duplicate Key Ignore
266
283
 
@@ -348,6 +365,29 @@ book.reload.title # => "Book1" (stayed the same)
348
365
  book.reload.author # => "Bob Barker" (changed)
349
366
  ```
350
367
 
368
+ PostgreSQL Using partial indexes
369
+
370
+ ```ruby
371
+ book = Book.create! title: "Book1", author: "George Orwell", published_at: Time.now
372
+ book.author = "Bob Barker"
373
+
374
+ # in migration
375
+ execute <<-SQL
376
+ CREATE INDEX books_published_at_index ON books (published_at) WHERE published_at IS NOT NULL;
377
+ SQL
378
+
379
+ # PostgreSQL version
380
+ Book.import [book], on_duplicate_key_update: {
381
+ conflict_target: [:id],
382
+ index_predicate: "published_at IS NOT NULL",
383
+ columns: [:author]
384
+ }
385
+
386
+ book.reload.title # => "Book1" (stayed the same)
387
+ book.reload.author # => "Bob Barker" (changed)
388
+ book.reload.published_at # => 2017-10-09 (stayed the same)
389
+ ```
390
+
351
391
  PostgreSQL Using constraints
352
392
 
353
393
  ```ruby
@@ -376,7 +416,7 @@ Book.import books, validate_uniqueness: true
376
416
 
377
417
  ### Return Info
378
418
 
379
- The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed`.
419
+ The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed.
380
420
 
381
421
  ```ruby
382
422
  articles = [
@@ -417,7 +457,9 @@ Should you wish to specify those columns, you may use the option `timestamps: fa
417
457
 
418
458
  However, it is also possible to set just `:created_at` in specific records. In this case despite using `timestamps: true`, `:created_at` will be updated only in records where that field is `nil`. Same rule applies for record associations when enabling the option `recursive: true`.
419
459
 
420
- If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is
460
+ If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is.
461
+ > **Note**
462
+ > If you are using ActiveRecord 7.0 or later, please use `ActiveRecord.default_timezone` instead.
421
463
 
422
464
  ### Callbacks
423
465
 
@@ -489,7 +531,8 @@ This allows an external gem to dynamically add an adapter without the need to ad
489
531
 
490
532
  ### Requiring
491
533
 
492
- Note: These instructions will only work if you are using version 0.2.0 or higher.
534
+ > **Note**
535
+ > These instructions will only work if you are using version 0.2.0 or higher.
493
536
 
494
537
  #### Autoloading via Bundler
495
538
 
@@ -526,11 +569,11 @@ require 'activerecord-import'
526
569
  ### Load Path Setup
527
570
  To understand how rubygems loads code you can reference the following:
528
571
 
529
- http://guides.rubygems.org/patterns/#loading_code
572
+ https://guides.rubygems.org/patterns/#loading-code
530
573
 
531
574
  And an example of how active_record dynamically load adapters:
532
575
 
533
- https://github.com/rails/rails/blob/master/activerecord/lib/active_record/connection_adapters/connection_specification.rb
576
+ https://github.com/rails/rails/blob/main/activerecord/lib/active_record/connection_adapters.rb
534
577
 
535
578
  In summary, when a gem is loaded rubygems adds the `lib` folder of the gem to the global load path `$LOAD_PATH` so that all `require` lookups will not propagate through all of the folders on the load path. When a `require` is issued each folder on the `$LOAD_PATH` is checked for the file and/or folder referenced. This allows a gem (like activerecord-import) to define push the activerecord-import folder (or namespace) on the `$LOAD_PATH` and any adapters provided by activerecord-import will be found by rubygems when the require is issued.
536
579
 
@@ -556,7 +599,7 @@ When rubygems pushes the `lib` folder onto the load path a `require` will now fi
556
599
 
557
600
  Activerecord-Import adds the `.import` method onto `ActiveRecord::Base`. There are other gems, such as `elasticsearch-rails`, that do the same thing. In conflicts such as this, there is an aliased method named `.bulk_import` that can be used interchangeably.
558
601
 
559
- If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by explcitly setting this value within the model. For example:
602
+ If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by explicitly setting this value within the model. For example:
560
603
 
561
604
  ```ruby
562
605
  class Post < ActiveRecord::Base
@@ -599,13 +642,26 @@ After that, you can run the tests. They run against multiple tests and ActiveRec
599
642
 
600
643
  This is one example of how to run the tests:
601
644
 
602
- ```ruby
645
+ ```bash
603
646
  rm Gemfile.lock
604
- AR_VERSION=4.2 bundle install
605
- AR_VERSION=4.2 bundle exec rake test:postgresql test:sqlite3 test:mysql2
647
+ AR_VERSION=7.0 bundle install
648
+ AR_VERSION=7.0 bundle exec rake test:postgresql test:sqlite3 test:mysql2
649
+ ```
650
+
651
+ Once you have pushed up your changes, you can find your CI results [here](https://github.com/zdennis/activerecord-import/actions).
652
+
653
+ #### Docker Setup
654
+
655
+ Before you begin, make sure you have [Docker](https://www.docker.com/products/docker-desktop/) and [Docker Compose](https://docs.docker.com/compose/) installed on your machine. If you don't, you can install both via Homebrew using the following command:
656
+
657
+ ```bash
658
+ brew install docker && brew install docker-compose
606
659
  ```
660
+ ##### Steps
607
661
 
608
- Once you have pushed up your changes, you can find your CI results [here](https://travis-ci.org/zdennis/activerecord-import/).
662
+ 1. In your terminal run `docker-compose up --build`
663
+ 1. In another tab/window run `docker-compose exec app bash`
664
+ 1. In that same terminal run the mysql2 test by running `bundle exec rake test:mysql2`
609
665
 
610
666
  ## Issue Triage [![Open Source Helpers](https://www.codetriage.com/zdennis/activerecord-import/badges/users.svg)](https://www.codetriage.com/zdennis/activerecord-import)
611
667
 
@@ -613,7 +669,7 @@ You can triage issues which may include reproducing bug reports or asking for vi
613
669
 
614
670
  # License
615
671
 
616
- This is licensed under the ruby license.
672
+ This is licensed under the MIT license.
617
673
 
618
674
  # Author
619
675
 
data/Rakefile CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "bundler"
2
4
  Bundler.setup
3
5
 
@@ -27,6 +29,7 @@ ADAPTERS = %w(
27
29
  sqlite3
28
30
  spatialite
29
31
  seamless_database_pool
32
+ trilogy
30
33
  ).freeze
31
34
  ADAPTERS.each do |adapter|
32
35
  namespace :test do
@@ -1,4 +1,5 @@
1
- # -*- encoding: utf-8 -*-
1
+ # frozen_string_literal: true
2
+
2
3
  require File.expand_path('../lib/activerecord-import/version', __FILE__)
3
4
 
4
5
  Gem::Specification.new do |gem|
@@ -6,8 +7,12 @@ Gem::Specification.new do |gem|
6
7
  gem.email = ["zach.dennis@gmail.com"]
7
8
  gem.summary = "Bulk insert extension for ActiveRecord"
8
9
  gem.description = "A library for bulk inserting data using ActiveRecord."
9
- gem.homepage = "http://github.com/zdennis/activerecord-import"
10
- gem.license = "Ruby"
10
+ gem.homepage = "https://github.com/zdennis/activerecord-import"
11
+ gem.license = "MIT"
12
+
13
+ gem.metadata = {
14
+ "changelog_uri" => "https://github.com/zdennis/activerecord-import/blob/master/CHANGELOG.md"
15
+ }
11
16
 
12
17
  gem.files = `git ls-files`.split($\)
13
18
  gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) }
@@ -16,8 +21,8 @@ Gem::Specification.new do |gem|
16
21
  gem.require_paths = ["lib"]
17
22
  gem.version = ActiveRecord::Import::VERSION
18
23
 
19
- gem.required_ruby_version = ">= 1.9.2"
24
+ gem.required_ruby_version = ">= 2.4.0"
20
25
 
21
- gem.add_runtime_dependency "activerecord", ">= 3.2"
26
+ gem.add_runtime_dependency "activerecord", ">= 4.2"
22
27
  gem.add_development_dependency "rake"
23
28
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'pathname'
2
4
  require "fileutils"
3
5
  require "active_record"
@@ -20,7 +22,11 @@ FileUtils.mkdir_p 'log'
20
22
  ActiveRecord::Base.configurations["test"] = YAML.load_file(File.join(benchmark_dir, "../test/database.yml"))[options.adapter]
21
23
  ActiveRecord::Base.logger = Logger.new("log/test.log")
22
24
  ActiveRecord::Base.logger.level = Logger::DEBUG
23
- ActiveRecord::Base.default_timezone = :utc
25
+ if ActiveRecord.respond_to?(:default_timezone)
26
+ ActiveRecord.default_timezone = :utc
27
+ else
28
+ ActiveRecord::Base.default_timezone = :utc
29
+ end
24
30
 
25
31
  require "activerecord-import"
26
32
  ActiveRecord::Base.establish_connection(:test)
@@ -35,11 +41,9 @@ require File.join(benchmark_dir, "../test/schema/generic_schema")
35
41
  adapter_schema = File.join(benchmark_dir, "schema/#{options.adapter}_schema.rb")
36
42
  require adapter_schema if File.exist?(adapter_schema)
37
43
 
38
- Dir[File.dirname(__FILE__) + "/models/*.rb"].each { |file| require file }
44
+ Dir["#{File.dirname(__FILE__)}/models/*.rb"].sort.each { |file| require file }
39
45
 
40
46
  require File.join( benchmark_dir, 'lib', "#{options.adapter}_benchmark" )
41
-
42
- table_types = nil
43
47
  table_types = if options.benchmark_all_types
44
48
  ["all"]
45
49
  else
@@ -47,8 +51,8 @@ else
47
51
  end
48
52
 
49
53
  letter = options.adapter[0].chr
50
- clazz_str = letter.upcase + options.adapter[1..-1].downcase
51
- clazz = Object.const_get( clazz_str + "Benchmark" )
54
+ clazz_str = letter.upcase + options.adapter[1..].downcase
55
+ clazz = Object.const_get( "#{clazz_str}Benchmark" )
52
56
 
53
57
  benchmarks = []
54
58
  options.number_of_objects.each do |num|
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  class BenchmarkBase
2
4
  attr_reader :results
3
5
 
@@ -14,7 +16,7 @@ class BenchmarkBase
14
16
  end
15
17
  end
16
18
 
17
- # Returns an OpenStruct which contains two attritues, +description+ and +tms+ after performing an
19
+ # Returns a struct which contains two attritues, +description+ and +tms+ after performing an
18
20
  # actual benchmark.
19
21
  #
20
22
  # == PARAMETERS
@@ -22,18 +24,21 @@ class BenchmarkBase
22
24
  # * blk - the block of code to benchmark
23
25
  #
24
26
  # == RETURNS
25
- # An OpenStruct object with the following attributes:
27
+ # A struct object with the following attributes:
26
28
  # * description - the description of the benchmark ran
27
29
  # * tms - a Benchmark::Tms containing the results of the benchmark
28
- def bm( description )
30
+
31
+ BmStruct = Struct.new( :description, :tms, :failed, keyword_init: true )
32
+
33
+ def bm( description, &block )
29
34
  tms = nil
30
35
  puts "Benchmarking #{description}"
31
36
 
32
- Benchmark.bm { |x| tms = x.report { yield } }
37
+ Benchmark.bm { |x| tms = x.report(&block) }
33
38
  delete_all
34
39
  failed = false
35
40
 
36
- OpenStruct.new description: description, tms: tms, failed: failed
41
+ BmStruct.new( description: description, tms: tms, failed: failed )
37
42
  end
38
43
 
39
44
  # Given a model class (ie: Topic), and an array of columns and value sets
@@ -1,5 +1,6 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'optparse'
2
- require 'ostruct'
3
4
 
4
5
  #
5
6
  # == PARAMETERS
@@ -27,7 +28,7 @@ module BenchmarkOptionParser
27
28
  print_valid_table_types( options, prefix: " " )
28
29
  end
29
30
 
30
- # TODO IMPLEMENT THIS
31
+ # TODO: IMPLEMENT THIS
31
32
  def self.print_valid_table_types( options, hsh = { prefix: '' } )
32
33
  if !options.table_types.keys.empty?
33
34
  options.table_types.keys.sort.each { |type| puts hsh[:prefix].to_s + type.to_s }
@@ -36,8 +37,11 @@ module BenchmarkOptionParser
36
37
  end
37
38
  end
38
39
 
40
+ OptionsStruct = Struct.new( :adapter, :table_types, :delete_on_finish, :number_of_objects, :outputs,
41
+ :benchmark_all_types, keyword_init: true )
42
+ OutputStruct = Struct.new( :format, :filename, keyword_init: true )
39
43
  def self.parse( args )
40
- options = OpenStruct.new(
44
+ options = OptionsStruct.new(
41
45
  adapter: 'mysql2',
42
46
  table_types: {},
43
47
  delete_on_finish: true,
@@ -79,12 +83,12 @@ module BenchmarkOptionParser
79
83
 
80
84
  # print results in CSV format
81
85
  opts.on( "--to-csv [String]", "Print results in a CSV file format" ) do |filename|
82
- options.outputs << OpenStruct.new( format: 'csv', filename: filename)
86
+ options.outputs << OutputStruct.new( format: 'csv', filename: filename)
83
87
  end
84
88
 
85
89
  # print results in HTML format
86
90
  opts.on( "--to-html [String]", "Print results in HTML format" ) do |filename|
87
- options.outputs << OpenStruct.new( format: 'html', filename: filename )
91
+ options.outputs << OutputStruct.new( format: 'html', filename: filename )
88
92
  end
89
93
  end # end opt.parse!
90
94
 
@@ -98,7 +102,7 @@ module BenchmarkOptionParser
98
102
  end
99
103
 
100
104
  options.number_of_objects = [1000] if options.number_of_objects.empty?
101
- options.outputs = [OpenStruct.new( format: 'html', filename: 'benchmark.html')] if options.outputs.empty?
105
+ options.outputs = [OutputStruct.new( format: 'html', filename: 'benchmark.html')] if options.outputs.empty?
102
106
 
103
107
  print_options( options )
104
108
 
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  # Taken from http://www.programmingishard.com/posts/show/128
2
4
  # Posted by rbates
3
5
  class Float
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  class Mysql2Benchmark < BenchmarkBase
2
4
  def benchmark_all( array_of_cols_and_vals )
3
5
  methods = self.methods.find_all { |m| m =~ /benchmark_/ }
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'csv'
2
4
 
3
5
  module OutputToCSV
@@ -1,7 +1,9 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'erb'
2
4
 
3
5
  module OutputToHTML
4
- TEMPLATE_HEADER = <<"EOT".freeze
6
+ TEMPLATE_HEADER = <<"EOT"
5
7
  <div>
6
8
  All times are rounded to the nearest thousandth for display purposes. Speedups next to each time are computed
7
9
  before any rounding occurs. Also, all speedup calculations are computed by comparing a given time against
@@ -9,7 +11,7 @@ module OutputToHTML
9
11
  </div>
10
12
  EOT
11
13
 
12
- TEMPLATE = <<"EOT".freeze
14
+ TEMPLATE = <<"EOT"
13
15
  <style>
14
16
  td#benchmarkTitle {
15
17
  border: 1px solid black;
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  class TestInnoDb < ActiveRecord::Base
2
4
  self.table_name = 'test_innodb'
3
5
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  class TestMemory < ActiveRecord::Base
2
4
  self.table_name = 'test_memory'
3
5
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  class TestMyISAM < ActiveRecord::Base
2
4
  self.table_name = 'test_myisam'
3
5
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  ActiveRecord::Schema.define do
2
4
  create_table :test_myisam, options: 'ENGINE=MyISAM', force: true do |t|
3
5
  t.column :my_name, :string, null: false
@@ -0,0 +1,34 @@
1
+ version: "3.5"
2
+
3
+ services:
4
+ mysql:
5
+ platform: linux/x86_64
6
+ image: mysql:5.7
7
+ volumes:
8
+ - mysql-data:/var/lib/mysql
9
+ ports:
10
+ - "3306:3306"
11
+
12
+ postgresql:
13
+ image: postgres:latest
14
+ volumes:
15
+ - postgresql-data:/var/lib/postgresql/data
16
+ ports:
17
+ - "5432:5432"
18
+
19
+ app:
20
+ build:
21
+ context: .
22
+ environment:
23
+ DB_HOST: mysql
24
+ AR_VERSION: 7.0
25
+ volumes:
26
+ - .:/usr/src/app
27
+ depends_on:
28
+ - mysql
29
+ - postgresql
30
+ command: tail -f /dev/null
31
+
32
+ volumes:
33
+ mysql-data:
34
+ postgresql-data:
data/gemfiles/5.2.gemfile CHANGED
@@ -1,2 +1,4 @@
1
+ # frozen_string_literal: true
2
+
1
3
  gem 'activerecord', '~> 5.2.0'
2
4
  gem 'composite_primary_keys', '~> 11.0'
data/gemfiles/6.0.gemfile CHANGED
@@ -1 +1,4 @@
1
+ # frozen_string_literal: true
2
+
1
3
  gem 'activerecord', '~> 6.0.0'
4
+ gem 'composite_primary_keys', '~> 12.0'
data/gemfiles/6.1.gemfile CHANGED
@@ -1 +1,4 @@
1
- gem 'activerecord', '~> 6.1.0.alpha', github: "rails/rails"
1
+ # frozen_string_literal: true
2
+
3
+ gem 'activerecord', '~> 6.1.0'
4
+ gem 'composite_primary_keys', '~> 13.0'
@@ -0,0 +1,4 @@
1
+ # frozen_string_literal: true
2
+
3
+ gem 'activerecord', '~> 7.0.0'
4
+ gem 'composite_primary_keys', '~> 14.0'
@@ -0,0 +1,3 @@
1
+ # frozen_string_literal: true
2
+
3
+ gem 'activerecord', '~> 7.1.0'
@@ -0,0 +1,3 @@
1
+ # frozen_string_literal: true
2
+
3
+ gem 'activerecord', '~> 7.2.0'
@@ -0,0 +1,3 @@
1
+ # frozen_string_literal: true
2
+
3
+ gem 'activerecord', '~> 8.0.0'
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "activerecord-import/adapters/abstract_adapter"
2
4
 
3
5
  module ActiveRecord # :nodoc:
@@ -1,6 +1,8 @@
1
- require "active_record/connection_adapters/mysql_adapter"
2
- require "activerecord-import/adapters/mysql_adapter"
1
+ # frozen_string_literal: true
3
2
 
4
- class ActiveRecord::ConnectionAdapters::MysqlAdapter
5
- include ActiveRecord::Import::MysqlAdapter
3
+ require "active_record/connection_adapters/mysql2_adapter"
4
+ require "activerecord-import/adapters/mysql2_adapter"
5
+
6
+ class ActiveRecord::ConnectionAdapters::Mysql2Adapter
7
+ include ActiveRecord::Import::Mysql2Adapter
6
8
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "active_record/connection_adapters/postgresql_adapter"
2
4
  require "activerecord-import/adapters/postgresql_adapter"
3
5
 
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "active_record/connection_adapters/sqlite3_adapter"
2
4
  require "activerecord-import/adapters/sqlite3_adapter"
3
5