activerecord-import 0.14.1 → 1.4.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (131) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/test.yaml +107 -0
  3. data/.gitignore +1 -0
  4. data/.rubocop.yml +74 -8
  5. data/Brewfile +3 -1
  6. data/CHANGELOG.md +448 -2
  7. data/Gemfile +26 -19
  8. data/LICENSE +21 -56
  9. data/README.markdown +568 -32
  10. data/Rakefile +5 -1
  11. data/activerecord-import.gemspec +8 -7
  12. data/benchmarks/README +2 -2
  13. data/benchmarks/benchmark.rb +8 -1
  14. data/benchmarks/lib/base.rb +2 -0
  15. data/benchmarks/lib/cli_parser.rb +5 -2
  16. data/benchmarks/lib/float.rb +2 -0
  17. data/benchmarks/lib/mysql2_benchmark.rb +2 -0
  18. data/benchmarks/lib/output_to_csv.rb +2 -0
  19. data/benchmarks/lib/output_to_html.rb +4 -2
  20. data/benchmarks/models/test_innodb.rb +2 -0
  21. data/benchmarks/models/test_memory.rb +2 -0
  22. data/benchmarks/models/test_myisam.rb +2 -0
  23. data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
  24. data/gemfiles/4.2.gemfile +4 -3
  25. data/gemfiles/5.0.gemfile +4 -3
  26. data/gemfiles/5.1.gemfile +4 -0
  27. data/gemfiles/5.2.gemfile +4 -0
  28. data/gemfiles/6.0.gemfile +4 -0
  29. data/gemfiles/6.1.gemfile +4 -0
  30. data/gemfiles/7.0.gemfile +4 -0
  31. data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
  32. data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
  33. data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
  34. data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +8 -0
  35. data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
  36. data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
  37. data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
  38. data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
  39. data/lib/activerecord-import/adapters/abstract_adapter.rb +12 -16
  40. data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
  41. data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
  42. data/lib/activerecord-import/adapters/mysql_adapter.rb +35 -18
  43. data/lib/activerecord-import/adapters/postgresql_adapter.rb +107 -28
  44. data/lib/activerecord-import/adapters/sqlite3_adapter.rb +141 -18
  45. data/lib/activerecord-import/base.rb +15 -8
  46. data/lib/activerecord-import/import.rb +642 -178
  47. data/lib/activerecord-import/mysql2.rb +2 -0
  48. data/lib/activerecord-import/postgresql.rb +2 -0
  49. data/lib/activerecord-import/sqlite3.rb +2 -0
  50. data/lib/activerecord-import/synchronize.rb +5 -3
  51. data/lib/activerecord-import/value_sets_parser.rb +29 -3
  52. data/lib/activerecord-import/version.rb +3 -1
  53. data/lib/activerecord-import.rb +5 -16
  54. data/test/adapters/jdbcmysql.rb +2 -0
  55. data/test/adapters/jdbcpostgresql.rb +2 -0
  56. data/test/adapters/jdbcsqlite3.rb +3 -0
  57. data/test/adapters/makara_postgis.rb +3 -0
  58. data/test/adapters/mysql2.rb +2 -0
  59. data/test/adapters/mysql2_makara.rb +2 -0
  60. data/test/adapters/mysql2spatial.rb +2 -0
  61. data/test/adapters/postgis.rb +2 -0
  62. data/test/adapters/postgresql.rb +2 -0
  63. data/test/adapters/postgresql_makara.rb +2 -0
  64. data/test/adapters/seamless_database_pool.rb +2 -0
  65. data/test/adapters/spatialite.rb +2 -0
  66. data/test/adapters/sqlite3.rb +2 -0
  67. data/test/{travis → github}/database.yml +7 -1
  68. data/test/import_test.rb +498 -32
  69. data/test/jdbcmysql/import_test.rb +2 -1
  70. data/test/jdbcpostgresql/import_test.rb +2 -1
  71. data/test/jdbcsqlite3/import_test.rb +6 -0
  72. data/test/makara_postgis/import_test.rb +10 -0
  73. data/test/models/account.rb +5 -0
  74. data/test/models/alarm.rb +4 -0
  75. data/test/models/animal.rb +8 -0
  76. data/test/models/bike_maker.rb +9 -0
  77. data/test/models/book.rb +4 -0
  78. data/test/models/car.rb +5 -0
  79. data/test/models/card.rb +5 -0
  80. data/test/models/chapter.rb +2 -0
  81. data/test/models/customer.rb +8 -0
  82. data/test/models/deck.rb +8 -0
  83. data/test/models/dictionary.rb +6 -0
  84. data/test/models/discount.rb +2 -0
  85. data/test/models/end_note.rb +2 -0
  86. data/test/models/group.rb +2 -0
  87. data/test/models/order.rb +8 -0
  88. data/test/models/playing_card.rb +4 -0
  89. data/test/models/promotion.rb +2 -0
  90. data/test/models/question.rb +2 -0
  91. data/test/models/rule.rb +2 -0
  92. data/test/models/tag.rb +7 -0
  93. data/test/models/tag_alias.rb +5 -0
  94. data/test/models/topic.rb +16 -0
  95. data/test/models/user.rb +5 -0
  96. data/test/models/user_token.rb +6 -0
  97. data/test/models/vendor.rb +9 -0
  98. data/test/models/widget.rb +18 -0
  99. data/test/mysql2/import_test.rb +2 -0
  100. data/test/mysql2_makara/import_test.rb +2 -0
  101. data/test/mysqlspatial2/import_test.rb +2 -0
  102. data/test/postgis/import_test.rb +6 -0
  103. data/test/postgresql/import_test.rb +2 -4
  104. data/test/schema/generic_schema.rb +88 -3
  105. data/test/schema/jdbcpostgresql_schema.rb +3 -0
  106. data/test/schema/mysql2_schema.rb +21 -0
  107. data/test/schema/postgis_schema.rb +3 -0
  108. data/test/schema/postgresql_schema.rb +63 -0
  109. data/test/schema/sqlite3_schema.rb +15 -0
  110. data/test/schema/version.rb +2 -0
  111. data/test/sqlite3/import_test.rb +4 -50
  112. data/test/support/active_support/test_case_extensions.rb +8 -1
  113. data/test/support/assertions.rb +2 -0
  114. data/test/support/factories.rb +17 -8
  115. data/test/support/generate.rb +10 -8
  116. data/test/support/mysql/import_examples.rb +17 -3
  117. data/test/support/postgresql/import_examples.rb +442 -9
  118. data/test/support/shared_examples/on_duplicate_key_ignore.rb +45 -0
  119. data/test/support/shared_examples/on_duplicate_key_update.rb +278 -1
  120. data/test/support/shared_examples/recursive_import.rb +137 -12
  121. data/test/support/sqlite3/import_examples.rb +232 -0
  122. data/test/synchronize_test.rb +10 -0
  123. data/test/test_helper.rb +44 -3
  124. data/test/value_sets_bytes_parser_test.rb +15 -2
  125. data/test/value_sets_records_parser_test.rb +2 -0
  126. metadata +74 -22
  127. data/.travis.yml +0 -52
  128. data/gemfiles/3.2.gemfile +0 -3
  129. data/gemfiles/4.0.gemfile +0 -3
  130. data/gemfiles/4.1.gemfile +0 -3
  131. data/test/schema/mysql_schema.rb +0 -16
data/README.markdown CHANGED
@@ -1,6 +1,6 @@
1
- # activerecord-import [![Build Status](https://travis-ci.org/zdennis/activerecord-import.svg?branch=master)](https://travis-ci.org/zdennis/activerecord-import)
1
+ # Activerecord-Import ![Build Status](https://github.com/zdennis/activerecord-import/actions/workflows/test.yaml/badge.svg)
2
2
 
3
- activerecord-import is a library for bulk inserting data using ActiveRecord.
3
+ Activerecord-Import is a library for bulk inserting data using ActiveRecord.
4
4
 
5
5
  One of its major features is following activerecord associations and generating the minimal
6
6
  number of SQL insert statements required, avoiding the N+1 insert problem. An example probably
@@ -21,46 +21,535 @@ and then the reviews:
21
21
  That would be about 4M SQL insert statements vs 3, which results in vastly improved performance. In our case, it converted
22
22
  an 18 hour batch process to <2 hrs.
23
23
 
24
- ### Rails 5.0
24
+ The gem provides the following high-level features:
25
+
26
+ * Works with raw columns and arrays of values (fastest)
27
+ * Works with model objects (faster)
28
+ * Performs validations (fast)
29
+ * Performs on duplicate key updates (requires MySQL, SQLite 3.24.0+, or Postgres 9.5+)
30
+
31
+ ## Table of Contents
32
+
33
+ * [Examples](#examples)
34
+ * [Introduction](#introduction)
35
+ * [Columns and Arrays](#columns-and-arrays)
36
+ * [Hashes](#hashes)
37
+ * [ActiveRecord Models](#activerecord-models)
38
+ * [Batching](#batching)
39
+ * [Recursive](#recursive)
40
+ * [Options](#options)
41
+ * [Duplicate Key Ignore](#duplicate-key-ignore)
42
+ * [Duplicate Key Update](#duplicate-key-update)
43
+ * [Return Info](#return-info)
44
+ * [Counter Cache](#counter-cache)
45
+ * [ActiveRecord Timestamps](#activerecord-timestamps)
46
+ * [Callbacks](#callbacks)
47
+ * [Supported Adapters](#supported-adapters)
48
+ * [Additional Adapters](#additional-adapters)
49
+ * [Requiring](#requiring)
50
+ * [Autoloading via Bundler](#autoloading-via-bundler)
51
+ * [Manually Loading](#manually-loading)
52
+ * [Load Path Setup](#load-path-setup)
53
+ * [Conflicts With Other Gems](#conflicts-with-other-gems)
54
+ * [More Information](#more-information)
55
+ * [Contributing](#contributing)
56
+ * [Running Tests](#running-tests)
57
+ * [Issue Triage](#issue-triage)
58
+
59
+ ### Examples
60
+
61
+ #### Introduction
62
+
63
+ This gem adds an `import` method (or `bulk_import`, for compatibility with gems like `elasticsearch-model`; see [Conflicts With Other Gems](#conflicts-with-other-gems)) to ActiveRecord classes.
64
+
65
+ Without `activerecord-import`, you'd write something like this:
25
66
 
26
- Use activerecord-import 0.11.0 or higher.
67
+ ```ruby
68
+ 10.times do |i|
69
+ Book.create! name: "book #{i}"
70
+ end
71
+ ```
72
+
73
+ This would end up making 10 SQL calls. YUCK! With `activerecord-import`, you can instead do this:
74
+
75
+ ```ruby
76
+ books = []
77
+ 10.times do |i|
78
+ books << Book.new(name: "book #{i}")
79
+ end
80
+ Book.import books # or use import!
81
+ ```
82
+
83
+ and only have 1 SQL call. Much better!
84
+
85
+ #### Columns and Arrays
86
+
87
+ The `import` method can take an array of column names (string or symbols) and an array of arrays. Each child array represents an individual record and its list of values in the same order as the columns. This is the fastest import mechanism and also the most primitive.
88
+
89
+ ```ruby
90
+ columns = [ :title, :author ]
91
+ values = [ ['Book1', 'George Orwell'], ['Book2', 'Bob Jones'] ]
92
+
93
+ # Importing without model validations
94
+ Book.import columns, values, validate: false
95
+
96
+ # Import with model validations
97
+ Book.import columns, values, validate: true
98
+
99
+ # when not specified :validate defaults to true
100
+ Book.import columns, values
101
+ ```
102
+
103
+ #### Hashes
104
+
105
+ The `import` method can take an array of hashes. The keys map to the column names in the database.
106
+
107
+ ```ruby
108
+ values = [{ title: 'Book1', author: 'George Orwell' }, { title: 'Book2', author: 'Bob Jones'}]
109
+
110
+ # Importing without model validations
111
+ Book.import values, validate: false
112
+
113
+ # Import with model validations
114
+ Book.import values, validate: true
115
+
116
+ # when not specified :validate defaults to true
117
+ Book.import values
118
+ ```
119
+ #### Import Using Hashes and Explicit Column Names
120
+
121
+ The `import` method can take an array of column names and an array of hash objects. The column names are used to determine what fields of data should be imported. The following example will only import books with the `title` field:
122
+
123
+ ```ruby
124
+ books = [
125
+ { title: "Book 1", author: "George Orwell" },
126
+ { title: "Book 2", author: "Bob Jones" }
127
+ ]
128
+ columns = [ :title ]
129
+
130
+ # without validations
131
+ Book.import columns, books, validate: false
132
+
133
+ # with validations
134
+ Book.import columns, books, validate: true
135
+
136
+ # when not specified :validate defaults to true
137
+ Book.import columns, books
138
+
139
+ # result in table books
140
+ # title | author
141
+ #--------|--------
142
+ # Book 1 | NULL
143
+ # Book 2 | NULL
144
+
145
+ ```
146
+
147
+ Using hashes will only work if the columns are consistent in every hash of the array. If this does not hold, an exception will be raised. There are two workarounds: use the array to instantiate an array of ActiveRecord objects and then pass that into `import` or divide the array into multiple ones with consistent columns and import each one separately.
148
+
149
+ See https://github.com/zdennis/activerecord-import/issues/507 for discussion.
150
+
151
+ ```ruby
152
+ arr = [
153
+ { bar: 'abc' },
154
+ { baz: 'xyz' },
155
+ { bar: '123', baz: '456' }
156
+ ]
157
+
158
+ # An exception will be raised
159
+ Foo.import arr
160
+
161
+ # better
162
+ arr.map! { |args| Foo.new(args) }
163
+ Foo.import arr
164
+
165
+ # better
166
+ arr.group_by(&:keys).each_value do |v|
167
+ Foo.import v
168
+ end
169
+ ```
170
+
171
+ #### ActiveRecord Models
172
+
173
+ The `import` method can take an array of models. The attributes will be pulled off from each model by looking at the columns available on the model.
174
+
175
+ ```ruby
176
+ books = [
177
+ Book.new(title: "Book 1", author: "George Orwell"),
178
+ Book.new(title: "Book 2", author: "Bob Jones")
179
+ ]
180
+
181
+ # without validations
182
+ Book.import books, validate: false
183
+
184
+ # with validations
185
+ Book.import books, validate: true
186
+
187
+ # when not specified :validate defaults to true
188
+ Book.import books
189
+ ```
190
+
191
+ The `import` method can take an array of column names and an array of models. The column names are used to determine what fields of data should be imported. The following example will only import books with the `title` field:
192
+
193
+ ```ruby
194
+ books = [
195
+ Book.new(title: "Book 1", author: "George Orwell"),
196
+ Book.new(title: "Book 2", author: "Bob Jones")
197
+ ]
198
+ columns = [ :title ]
199
+
200
+ # without validations
201
+ Book.import columns, books, validate: false
27
202
 
28
- ### Rails 4.0
203
+ # with validations
204
+ Book.import columns, books, validate: true
29
205
 
30
- Use activerecord-import 0.4.0 or higher.
206
+ # when not specified :validate defaults to true
207
+ Book.import columns, books
31
208
 
32
- ### Rails 3.1.x up to, but not including 4.0
209
+ # result in table books
210
+ # title | author
211
+ #--------|--------
212
+ # Book 1 | NULL
213
+ # Book 2 | NULL
33
214
 
34
- Use the latest in the activerecord-import 0.3.x series.
215
+ ```
216
+
217
+ #### Batching
218
+
219
+ The `import` method can take a `batch_size` option to control the number of rows to insert per INSERT statement. The default is the total number of records being inserted so there is a single INSERT statement.
220
+
221
+ ```ruby
222
+ books = [
223
+ Book.new(title: "Book 1", author: "George Orwell"),
224
+ Book.new(title: "Book 2", author: "Bob Jones"),
225
+ Book.new(title: "Book 1", author: "John Doe"),
226
+ Book.new(title: "Book 2", author: "Richard Wright")
227
+ ]
228
+ columns = [ :title ]
229
+
230
+ # 2 INSERT statements for 4 records
231
+ Book.import columns, books, batch_size: 2
232
+ ```
233
+
234
+ If your import is particularly large or slow (possibly due to [callbacks](#callbacks)) whilst batch importing, you might want a way to report back on progress. This is supported by passing a callable as the `batch_progress` option. e.g:
235
+
236
+ ```ruby
237
+ my_proc = ->(rows_size, num_batches, current_batch_number, batch_duration_in_secs) {
238
+ # Using the arguments provided to the callable, you can
239
+ # send an email, post to a websocket,
240
+ # update slack, alert if import is taking too long, etc.
241
+ }
242
+
243
+ Book.import columns, books, batch_size: 2, batch_progress: my_proc
244
+ ```
245
+
246
+ #### Recursive
247
+
248
+ NOTE: This only works with PostgreSQL and ActiveRecord objects. This won't work with
249
+ hashes or arrays as recursive inputs.
250
+
251
+ Assume that Books <code>has_many</code> Reviews.
252
+
253
+ ```ruby
254
+ books = []
255
+ 10.times do |i|
256
+ book = Book.new(name: "book #{i}")
257
+ book.reviews.build(title: "Excellent")
258
+ books << book
259
+ end
260
+ Book.import books, recursive: true
261
+ ```
262
+
263
+ ### Options
264
+
265
+ Key | Options | Default | Description
266
+ ------------------------- | --------------------- | ------------------ | -----------
267
+ :validate | `true`/`false` | `true` | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
268
+ :validate_uniqueness | `true`/`false` | `false` | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
269
+ :validate_with_context | `Symbol` |`:create`/`:update` | Allows passing an ActiveModel validation context for each model. Default is `:create` for new records and `:update` for existing ones.
270
+ :track_validation_failures| `true`/`false` | `false` | When this is set to true, `failed_instances` will be an array of arrays, with each inner array having the form `[:index_in_dataset, :object_with_errors]`
271
+ :on_duplicate_key_ignore | `true`/`false` | `false` | Allows skipping records with duplicate keys. See [here](#duplicate-key-ignore) for more details.
272
+ :ignore | `true`/`false` | `false` | Alias for :on_duplicate_key_ignore.
273
+ :on_duplicate_key_update | :all, `Array`, `Hash` | N/A | Allows upsert logic to be used. See [here](#duplicate-key-update) for more details.
274
+ :synchronize | `Array` | N/A | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
275
+ :timestamps | `true`/`false` | `true` | Enables/disables timestamps on imported records.
276
+ :recursive | `true`/`false` | `false` | Imports has_many/has_one associations (PostgreSQL only).
277
+ :batch_size | `Integer` | total # of records | Max number of records to insert per import
278
+ :raise_error | `true`/`false` | `false` | Raises an exception at the first invalid record. This means there will not be a result object returned. The `import!` method is a shortcut for this.
279
+ :all_or_none | `true`/`false` | `false` | Will not import any records if there is a record with validation errors.
280
+
281
+ #### Duplicate Key Ignore
282
+
283
+ [MySQL](http://dev.mysql.com/doc/refman/5.0/en/insert-on-duplicate.html), [SQLite](https://www.sqlite.org/lang_insert.html), and [PostgreSQL](https://www.postgresql.org/docs/current/static/sql-insert.html#SQL-ON-CONFLICT) (9.5+) support `on_duplicate_key_ignore` which allows you to skip records if a primary or unique key constraint is violated.
284
+
285
+ For Postgres 9.5+ it adds `ON CONFLICT DO NOTHING`, for MySQL it uses `INSERT IGNORE`, and for SQLite it uses `INSERT OR IGNORE`. Cannot be enabled on a recursive import. For database adapters that normally support setting primary keys on imported objects, this option prevents that from occurring.
286
+
287
+ ```ruby
288
+ book = Book.create! title: "Book1", author: "George Orwell"
289
+ book.title = "Updated Book Title"
290
+ book.author = "Bob Barker"
291
+
292
+ Book.import [book], on_duplicate_key_ignore: true
35
293
 
36
- ### Rails 3.0.x up to, but not including 3.1
294
+ book.reload.title # => "Book1" (stayed the same)
295
+ book.reload.author # => "George Orwell" (stayed the same)
296
+ ```
297
+
298
+ The option `:on_duplicate_key_ignore` is bypassed when `:recursive` is enabled for [PostgreSQL imports](https://github.com/zdennis/activerecord-import/wiki#recursive-example-postgresql-only).
299
+
300
+ #### Duplicate Key Update
301
+
302
+ MySQL, PostgreSQL (9.5+), and SQLite (3.24.0+) support `on duplicate key update` (also known as "upsert") which allows you to specify fields whose values should be updated if a primary or unique key constraint is violated.
303
+
304
+ One big difference between MySQL and PostgreSQL support is that MySQL will handle any conflict that happens, but PostgreSQL requires that you specify which columns the conflict would occur over. SQLite models its upsert support after PostgreSQL.
305
+
306
+ This will use MySQL's `ON DUPLICATE KEY UPDATE` or Postgres/SQLite `ON CONFLICT DO UPDATE` to do upsert.
307
+
308
+ Basic Update
309
+
310
+ ```ruby
311
+ book = Book.create! title: "Book1", author: "George Orwell"
312
+ book.title = "Updated Book Title"
313
+ book.author = "Bob Barker"
314
+
315
+ # MySQL version
316
+ Book.import [book], on_duplicate_key_update: [:title]
317
+
318
+ # PostgreSQL version
319
+ Book.import [book], on_duplicate_key_update: {conflict_target: [:id], columns: [:title]}
320
+
321
+ # PostgreSQL shorthand version (conflict target must be primary key)
322
+ Book.import [book], on_duplicate_key_update: [:title]
323
+
324
+ book.reload.title # => "Updated Book Title" (changed)
325
+ book.reload.author # => "George Orwell" (stayed the same)
326
+ ```
327
+
328
+ Using the value from another column
329
+
330
+ ```ruby
331
+ book = Book.create! title: "Book1", author: "George Orwell"
332
+ book.title = "Updated Book Title"
333
+
334
+ # MySQL version
335
+ Book.import [book], on_duplicate_key_update: {author: :title}
336
+
337
+ # PostgreSQL version (no shorthand version)
338
+ Book.import [book], on_duplicate_key_update: {
339
+ conflict_target: [:id], columns: {author: :title}
340
+ }
341
+
342
+ book.reload.title # => "Book1" (stayed the same)
343
+ book.reload.author # => "Updated Book Title" (changed)
344
+ ```
345
+
346
+ Using Custom SQL
347
+
348
+ ```ruby
349
+ book = Book.create! title: "Book1", author: "George Orwell"
350
+ book.author = "Bob Barker"
37
351
 
38
- Use activerecord-import 0.2.11. As of activerecord-import 0.3.0 we are relying on functionality that was introduced in Rails 3.1. Since Rails 3.0.x is no longer a supported version of Rails we have decided to drop support as well.
352
+ # MySQL version
353
+ Book.import [book], on_duplicate_key_update: "author = values(author)"
39
354
 
40
- ### For More Information
355
+ # PostgreSQL version
356
+ Book.import [book], on_duplicate_key_update: {
357
+ conflict_target: [:id], columns: "author = excluded.author"
358
+ }
41
359
 
42
- For more information on activerecord-import please see its wiki: https://github.com/zdennis/activerecord-import/wiki
360
+ # PostgreSQL shorthand version (conflict target must be primary key)
361
+ Book.import [book], on_duplicate_key_update: "author = excluded.author"
362
+
363
+ book.reload.title # => "Book1" (stayed the same)
364
+ book.reload.author # => "Bob Barker" (changed)
365
+ ```
366
+
367
+ PostgreSQL Using constraints
368
+
369
+ ```ruby
370
+ book = Book.create! title: "Book1", author: "George Orwell", edition: 3, published_at: nil
371
+ book.published_at = Time.now
372
+
373
+ # in migration
374
+ execute <<-SQL
375
+ ALTER TABLE books
376
+ ADD CONSTRAINT for_upsert UNIQUE (title, author, edition);
377
+ SQL
378
+
379
+ # PostgreSQL version
380
+ Book.import [book], on_duplicate_key_update: {constraint_name: :for_upsert, columns: [:published_at]}
381
+
382
+
383
+ book.reload.title # => "Book1" (stayed the same)
384
+ book.reload.author # => "George Orwell" (stayed the same)
385
+ book.reload.edition # => 3 (stayed the same)
386
+ book.reload.published_at # => 2017-10-09 (changed)
387
+ ```
388
+
389
+ ```ruby
390
+ Book.import books, validate_uniqueness: true
391
+ ```
392
+
393
+ ### Return Info
394
+
395
+ The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed.
396
+
397
+ ```ruby
398
+ articles = [
399
+ Article.new(author_id: 1, title: 'First Article', content: 'This is the first article'),
400
+ Article.new(author_id: 2, title: 'Second Article', content: ''),
401
+ Article.new(author_id: 3, content: '')
402
+ ]
403
+
404
+ demo = Article.import(articles, returning: :title) # => #<struct ActiveRecord::Import::Result
405
+
406
+ demo.failed_instances
407
+ => [#<Article id: 3, author_id: 3, title: nil, content: "", created_at: nil, updated_at: nil>]
408
+
409
+ demo.num_inserts
410
+ => 1,
411
+
412
+ demo.ids
413
+ => ["1", "2"] # for Postgres
414
+ => [] # for other DBs
415
+
416
+ demo.results
417
+ => ["First Article", "Second Article"] # for Postgres
418
+ => [] # for other DBs
419
+ ```
420
+
421
+ ### Counter Cache
422
+
423
+ When running `import`, `activerecord-import` does not automatically update counter cache columns. To update these columns, you will need to do one of the following:
424
+
425
+ * Provide values to the column as an argument on your object that is passed in.
426
+ * Manually update the column after the record has been imported.
427
+
428
+ ### ActiveRecord Timestamps
429
+
430
+ If you're familiar with ActiveRecord you're probably familiar with its timestamp columns: created_at, created_on, updated_at, updated_on, etc. When importing data the timestamp fields will continue to work as expected and each timestamp column will be set.
431
+
432
+ Should you wish to specify those columns, you may use the option `timestamps: false`.
433
+
434
+ However, it is also possible to set just `:created_at` in specific records. In this case despite using `timestamps: true`, `:created_at` will be updated only in records where that field is `nil`. Same rule applies for record associations when enabling the option `recursive: true`.
435
+
436
+ If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is.
437
+ NOTE: If you are using ActiveRecord 7.0 or later, please use `ActiveRecord.default_timezone` instead.
438
+
439
+ ### Callbacks
440
+
441
+ ActiveRecord callbacks related to [creating](http://guides.rubyonrails.org/active_record_callbacks.html#creating-an-object), [updating](http://guides.rubyonrails.org/active_record_callbacks.html#updating-an-object), or [destroying](http://guides.rubyonrails.org/active_record_callbacks.html#destroying-an-object) records (other than `before_validation` and `after_validation`) will NOT be called when calling the import method. This is because it is mass importing rows of data and doesn't necessarily have access to in-memory ActiveRecord objects.
442
+
443
+ If you do have a collection of in-memory ActiveRecord objects you can do something like this:
444
+
445
+ ```ruby
446
+ books.each do |book|
447
+ book.run_callbacks(:save) { false }
448
+ book.run_callbacks(:create) { false }
449
+ end
450
+ Book.import(books)
451
+ ```
452
+
453
+ This will run before_create and before_save callbacks on each item. The `false` argument is needed to prevent after_save being run, which wouldn't make sense prior to bulk import. Something to note in this example is that the before_create and before_save callbacks will run before the validation callbacks.
454
+
455
+ If that is an issue, another possible approach is to loop through your models first to do validations and then only run callbacks on and import the valid models.
456
+
457
+ ```ruby
458
+ valid_books = []
459
+ invalid_books = []
460
+
461
+ books.each do |book|
462
+ if book.valid?
463
+ valid_books << book
464
+ else
465
+ invalid_books << book
466
+ end
467
+ end
468
+
469
+ valid_books.each do |book|
470
+ book.run_callbacks(:save) { false }
471
+ book.run_callbacks(:create) { false }
472
+ end
473
+
474
+ Book.import valid_books, validate: false
475
+ ```
476
+
477
+ ### Supported Adapters
478
+
479
+ The following database adapters are currently supported:
480
+
481
+ * MySQL - supports core import functionality plus on duplicate key update support (included in activerecord-import 0.1.0 and higher)
482
+ * MySQL2 - supports core import functionality plus on duplicate key update support (included in activerecord-import 0.2.0 and higher)
483
+ * PostgreSQL - supports core import functionality (included in activerecord-import 0.1.0 and higher)
484
+ * SQLite3 - supports core import functionality (included in activerecord-import 0.1.0 and higher)
485
+ * Oracle - supports core import functionality through DML trigger (available as an external gem: [activerecord-import-oracle_enhanced](https://github.com/keeguon/activerecord-import-oracle_enhanced)
486
+ * SQL Server - supports core import functionality (available as an external gem: [activerecord-import-sqlserver](https://github.com/keeguon/activerecord-import-sqlserver)
487
+
488
+ If your adapter isn't listed here, please consider creating an external gem as described in the README to provide support. If you do, feel free to update this wiki to include a link to the new adapter's repository!
489
+
490
+ To test which features are supported by your adapter, use the following methods on a model class:
491
+ * `supports_import?(*args)`
492
+ * `supports_on_duplicate_key_update?`
493
+ * `supports_setting_primary_key_of_imported_objects?`
494
+
495
+ ### Additional Adapters
43
496
 
44
- ## Additional Adapters
45
497
  Additional adapters can be provided by gems external to activerecord-import by providing an adapter that matches the naming convention setup by activerecord-import (and subsequently activerecord) for dynamically loading adapters. This involves also providing a folder on the load path that follows the activerecord-import naming convention to allow activerecord-import to dynamically load the file.
46
498
 
47
499
  When `ActiveRecord::Import.require_adapter("fake_name")` is called the require will be:
48
500
 
49
501
  ```ruby
50
- require 'activerecord-import/active_record/adapters/fake_name_adapter'
502
+ require 'activerecord-import/active_record/adapters/fake_name_adapter'
51
503
  ```
52
504
 
53
- This allows an external gem to dyanmically add an adapter without the need to add any file/code to the core activerecord-import gem.
505
+ This allows an external gem to dynamically add an adapter without the need to add any file/code to the core activerecord-import gem.
506
+
507
+ ### Requiring
508
+
509
+ Note: These instructions will only work if you are using version 0.2.0 or higher.
510
+
511
+ #### Autoloading via Bundler
512
+
513
+ If you are using Rails or otherwise autoload your dependencies via Bundler, all you need to do add the gem to your `Gemfile` like so:
514
+
515
+ ```ruby
516
+ gem 'activerecord-import'
517
+ ```
518
+
519
+ #### Manually Loading
520
+
521
+ You may want to manually load activerecord-import for one reason or another. First, add the `require: false` argument like so:
522
+
523
+ ```ruby
524
+ gem 'activerecord-import', require: false
525
+ ```
526
+
527
+ This will allow you to load up activerecord-import in the file or files where you are using it and only load the parts you need.
528
+ If you are doing this within Rails and ActiveRecord has established a database connection (such as within a controller), you will need to do extra initialization work:
529
+
530
+ ```ruby
531
+ require 'activerecord-import/base'
532
+ # load the appropriate database adapter (postgresql, mysql2, sqlite3, etc)
533
+ require 'activerecord-import/active_record/adapters/postgresql_adapter'
534
+ ```
535
+
536
+ If your gem dependencies aren’t autoloaded, and your script will be establishing a database connection, then simply require activerecord-import after ActiveRecord has been loaded, i.e.:
537
+
538
+ ```ruby
539
+ require 'active_record'
540
+ require 'activerecord-import'
541
+ ```
54
542
 
55
543
  ### Load Path Setup
56
544
  To understand how rubygems loads code you can reference the following:
57
545
 
58
- http://guides.rubygems.org/patterns/#loading_code
546
+ http://guides.rubygems.org/patterns/#loading-code
59
547
 
60
548
  And an example of how active_record dynamically load adapters:
549
+
61
550
  https://github.com/rails/rails/blob/master/activerecord/lib/active_record/connection_adapters/connection_specification.rb
62
551
 
63
- In summary, when a gem is loaded rubygems adds the `lib` folder of the gem to the global load path `$LOAD_PATH` so that all `require` lookups will not propegate through all of the folders on the load path. When a `require` is issued each folder on the `$LOAD_PATH` is checked for the file and/or folder referenced. This allows a gem (like activerecord-import) to define push the activerecord-import folder (or namespace) on the `$LOAD_PATH` and any adapters provided by activerecord-import will be found by rubygems when the require is issued.
552
+ In summary, when a gem is loaded rubygems adds the `lib` folder of the gem to the global load path `$LOAD_PATH` so that all `require` lookups will not propagate through all of the folders on the load path. When a `require` is issued each folder on the `$LOAD_PATH` is checked for the file and/or folder referenced. This allows a gem (like activerecord-import) to define push the activerecord-import folder (or namespace) on the `$LOAD_PATH` and any adapters provided by activerecord-import will be found by rubygems when the require is issued.
64
553
 
65
554
  If `fake_name` adapter is needed by a gem (potentially called `activerecord-import-fake_name`) then the folder structure should look as follows:
66
555
 
@@ -68,34 +557,81 @@ If `fake_name` adapter is needed by a gem (potentially called `activerecord-impo
68
557
  activerecord-import-fake_name/
69
558
  |-- activerecord-import-fake_name.gemspec
70
559
  |-- lib
560
+ | |-- activerecord-import-fake_name.rb
71
561
  | |-- activerecord-import-fake_name
72
562
  | | |-- version.rb
73
563
  | |-- activerecord-import
74
564
  | | |-- active_record
75
565
  | | | |-- adapters
76
566
  | | | |-- fake_name_adapter.rb
77
- |--activerecord-import-fake_name.rb
78
567
  ```
79
568
 
80
569
  When rubygems pushes the `lib` folder onto the load path a `require` will now find `activerecord-import/active_record/adapters/fake_name_adapter` as it runs through the lookup process for a ruby file under that path in `$LOAD_PATH`
81
570
 
571
+
572
+ ### Conflicts With Other Gems
573
+
574
+ Activerecord-Import adds the `.import` method onto `ActiveRecord::Base`. There are other gems, such as `elasticsearch-rails`, that do the same thing. In conflicts such as this, there is an aliased method named `.bulk_import` that can be used interchangeably.
575
+
576
+ If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by explicitly setting this value within the model. For example:
577
+
578
+ ```ruby
579
+ class Post < ActiveRecord::Base
580
+ self.sequence_name = "posts_seq"
581
+ end
582
+ ```
583
+
584
+ Another way to work around the issue is to call `.reset_sequence_name` on the model. For example:
585
+
586
+ ```ruby
587
+ schemas.all.each do |schema|
588
+ Apartment::Tenant.switch! schema.name
589
+ ActiveRecord::Base.transaction do
590
+ Post.reset_sequence_name
591
+
592
+ Post.import posts
593
+ end
594
+ end
595
+ ```
596
+
597
+ See https://github.com/zdennis/activerecord-import/issues/233 for further discussion.
598
+
599
+ ### More Information
600
+
601
+ For more information on Activerecord-Import please see its wiki: https://github.com/zdennis/activerecord-import/wiki
602
+
603
+ To document new information, please add to the README instead of the wiki. See https://github.com/zdennis/activerecord-import/issues/397 for discussion.
604
+
605
+ ### Contributing
606
+
607
+ #### Running Tests
608
+
609
+ The first thing you need to do is set up your database(s):
610
+
611
+ * copy `test/database.yml.sample` to `test/database.yml`
612
+ * modify `test/database.yml` for your database settings
613
+ * create databases as needed
614
+
615
+ After that, you can run the tests. They run against multiple tests and ActiveRecord versions.
616
+
617
+ This is one example of how to run the tests:
618
+
619
+ ```bash
620
+ rm Gemfile.lock
621
+ AR_VERSION=7.0 bundle install
622
+ AR_VERSION=7.0 bundle exec rake test:postgresql test:sqlite3 test:mysql2
623
+ ```
624
+
625
+ Once you have pushed up your changes, you can find your CI results [here](https://github.com/zdennis/activerecord-import/actions).
626
+
627
+ ## Issue Triage [![Open Source Helpers](https://www.codetriage.com/zdennis/activerecord-import/badges/users.svg)](https://www.codetriage.com/zdennis/activerecord-import)
628
+
629
+ You can triage issues which may include reproducing bug reports or asking for vital information, such as version numbers or reproduction instructions. If you would like to start triaging issues, one easy way to get started is to [subscribe to activerecord-import on CodeTriage](https://www.codetriage.com/zdennis/activerecord-import).
630
+
82
631
  # License
83
632
 
84
- This is licensed under the ruby license.
633
+ This is licensed under the MIT license.
85
634
 
86
635
  # Author
87
636
 
88
637
  Zach Dennis (zach.dennis@gmail.com)
89
-
90
- # Contributors
91
-
92
- * Jordan Owens (@jkowens)
93
- * Erik Michaels-Ober (@sferik)
94
- * Blythe Dunham
95
- * Gabe da Silveira
96
- * Henry Work
97
- * James Herdman
98
- * Marcus Crafter
99
- * Thibaud Guillaume-Gentil
100
- * Mark Van Holstyn
101
- * Victor Costan
data/Rakefile CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "bundler"
2
4
  Bundler.setup
3
5
 
@@ -18,10 +20,12 @@ ADAPTERS = %w(
18
20
  mysql2_makara
19
21
  mysql2spatial
20
22
  jdbcmysql
23
+ jdbcsqlite3
21
24
  jdbcpostgresql
22
25
  postgresql
23
26
  postgresql_makara
24
27
  postgis
28
+ makara_postgis
25
29
  sqlite3
26
30
  spatialite
27
31
  seamless_database_pool
@@ -30,7 +34,7 @@ ADAPTERS.each do |adapter|
30
34
  namespace :test do
31
35
  desc "Runs #{adapter} database tests."
32
36
  Rake::TestTask.new(adapter) do |t|
33
- # FactoryGirl has an issue with warnings, so turn off, so noisy
37
+ # FactoryBot has an issue with warnings, so turn off, so noisy
34
38
  # t.warning = true
35
39
  t.test_files = FileList["test/adapters/#{adapter}.rb", "test/*_test.rb", "test/active_record/*_test.rb", "test/#{adapter}/**/*_test.rb"]
36
40
  end