activerecord-import 1.0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (123) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +32 -0
  3. data/.rubocop.yml +49 -0
  4. data/.rubocop_todo.yml +36 -0
  5. data/.travis.yml +74 -0
  6. data/Brewfile +3 -0
  7. data/CHANGELOG.md +430 -0
  8. data/Gemfile +59 -0
  9. data/LICENSE +56 -0
  10. data/README.markdown +619 -0
  11. data/Rakefile +68 -0
  12. data/activerecord-import.gemspec +23 -0
  13. data/benchmarks/README +32 -0
  14. data/benchmarks/benchmark.rb +68 -0
  15. data/benchmarks/lib/base.rb +138 -0
  16. data/benchmarks/lib/cli_parser.rb +107 -0
  17. data/benchmarks/lib/float.rb +15 -0
  18. data/benchmarks/lib/mysql2_benchmark.rb +19 -0
  19. data/benchmarks/lib/output_to_csv.rb +19 -0
  20. data/benchmarks/lib/output_to_html.rb +64 -0
  21. data/benchmarks/models/test_innodb.rb +3 -0
  22. data/benchmarks/models/test_memory.rb +3 -0
  23. data/benchmarks/models/test_myisam.rb +3 -0
  24. data/benchmarks/schema/mysql_schema.rb +16 -0
  25. data/gemfiles/3.2.gemfile +2 -0
  26. data/gemfiles/4.0.gemfile +2 -0
  27. data/gemfiles/4.1.gemfile +2 -0
  28. data/gemfiles/4.2.gemfile +2 -0
  29. data/gemfiles/5.0.gemfile +2 -0
  30. data/gemfiles/5.1.gemfile +2 -0
  31. data/gemfiles/5.2.gemfile +2 -0
  32. data/gemfiles/6.0.gemfile +1 -0
  33. data/gemfiles/6.1.gemfile +1 -0
  34. data/lib/activerecord-import.rb +6 -0
  35. data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +9 -0
  36. data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -0
  37. data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +6 -0
  38. data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +6 -0
  39. data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +6 -0
  40. data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +6 -0
  41. data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +7 -0
  42. data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +6 -0
  43. data/lib/activerecord-import/adapters/abstract_adapter.rb +66 -0
  44. data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +5 -0
  45. data/lib/activerecord-import/adapters/mysql2_adapter.rb +5 -0
  46. data/lib/activerecord-import/adapters/mysql_adapter.rb +129 -0
  47. data/lib/activerecord-import/adapters/postgresql_adapter.rb +217 -0
  48. data/lib/activerecord-import/adapters/sqlite3_adapter.rb +180 -0
  49. data/lib/activerecord-import/base.rb +43 -0
  50. data/lib/activerecord-import/import.rb +1059 -0
  51. data/lib/activerecord-import/mysql2.rb +7 -0
  52. data/lib/activerecord-import/postgresql.rb +7 -0
  53. data/lib/activerecord-import/sqlite3.rb +7 -0
  54. data/lib/activerecord-import/synchronize.rb +66 -0
  55. data/lib/activerecord-import/value_sets_parser.rb +77 -0
  56. data/lib/activerecord-import/version.rb +5 -0
  57. data/test/adapters/jdbcmysql.rb +1 -0
  58. data/test/adapters/jdbcpostgresql.rb +1 -0
  59. data/test/adapters/jdbcsqlite3.rb +1 -0
  60. data/test/adapters/makara_postgis.rb +1 -0
  61. data/test/adapters/mysql2.rb +1 -0
  62. data/test/adapters/mysql2_makara.rb +1 -0
  63. data/test/adapters/mysql2spatial.rb +1 -0
  64. data/test/adapters/postgis.rb +1 -0
  65. data/test/adapters/postgresql.rb +1 -0
  66. data/test/adapters/postgresql_makara.rb +1 -0
  67. data/test/adapters/seamless_database_pool.rb +1 -0
  68. data/test/adapters/spatialite.rb +1 -0
  69. data/test/adapters/sqlite3.rb +1 -0
  70. data/test/database.yml.sample +52 -0
  71. data/test/import_test.rb +903 -0
  72. data/test/jdbcmysql/import_test.rb +5 -0
  73. data/test/jdbcpostgresql/import_test.rb +4 -0
  74. data/test/jdbcsqlite3/import_test.rb +4 -0
  75. data/test/makara_postgis/import_test.rb +8 -0
  76. data/test/models/account.rb +3 -0
  77. data/test/models/alarm.rb +2 -0
  78. data/test/models/bike_maker.rb +7 -0
  79. data/test/models/book.rb +9 -0
  80. data/test/models/car.rb +3 -0
  81. data/test/models/chapter.rb +4 -0
  82. data/test/models/dictionary.rb +4 -0
  83. data/test/models/discount.rb +3 -0
  84. data/test/models/end_note.rb +4 -0
  85. data/test/models/group.rb +3 -0
  86. data/test/models/promotion.rb +3 -0
  87. data/test/models/question.rb +3 -0
  88. data/test/models/rule.rb +3 -0
  89. data/test/models/tag.rb +4 -0
  90. data/test/models/topic.rb +23 -0
  91. data/test/models/user.rb +3 -0
  92. data/test/models/user_token.rb +4 -0
  93. data/test/models/vendor.rb +7 -0
  94. data/test/models/widget.rb +24 -0
  95. data/test/mysql2/import_test.rb +5 -0
  96. data/test/mysql2_makara/import_test.rb +6 -0
  97. data/test/mysqlspatial2/import_test.rb +6 -0
  98. data/test/postgis/import_test.rb +8 -0
  99. data/test/postgresql/import_test.rb +4 -0
  100. data/test/schema/generic_schema.rb +194 -0
  101. data/test/schema/jdbcpostgresql_schema.rb +1 -0
  102. data/test/schema/mysql2_schema.rb +19 -0
  103. data/test/schema/postgis_schema.rb +1 -0
  104. data/test/schema/postgresql_schema.rb +47 -0
  105. data/test/schema/sqlite3_schema.rb +13 -0
  106. data/test/schema/version.rb +10 -0
  107. data/test/sqlite3/import_test.rb +4 -0
  108. data/test/support/active_support/test_case_extensions.rb +75 -0
  109. data/test/support/assertions.rb +73 -0
  110. data/test/support/factories.rb +64 -0
  111. data/test/support/generate.rb +29 -0
  112. data/test/support/mysql/import_examples.rb +98 -0
  113. data/test/support/postgresql/import_examples.rb +563 -0
  114. data/test/support/shared_examples/on_duplicate_key_ignore.rb +43 -0
  115. data/test/support/shared_examples/on_duplicate_key_update.rb +368 -0
  116. data/test/support/shared_examples/recursive_import.rb +216 -0
  117. data/test/support/sqlite3/import_examples.rb +231 -0
  118. data/test/synchronize_test.rb +41 -0
  119. data/test/test_helper.rb +75 -0
  120. data/test/travis/database.yml +66 -0
  121. data/test/value_sets_bytes_parser_test.rb +104 -0
  122. data/test/value_sets_records_parser_test.rb +32 -0
  123. metadata +259 -0
data/Gemfile ADDED
@@ -0,0 +1,59 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
4
+
5
+ version = ENV['AR_VERSION'].to_f
6
+
7
+ mysql2_version = '0.3.0'
8
+ mysql2_version = '0.4.0' if version >= 4.2
9
+ sqlite3_version = '1.3.0'
10
+ sqlite3_version = '1.4.0' if version >= 6.0
11
+
12
+ group :development, :test do
13
+ gem 'rubocop', '~> 0.40.0'
14
+ gem 'rake'
15
+ end
16
+
17
+ # Database Adapters
18
+ platforms :ruby do
19
+ gem "mysql2", "~> #{mysql2_version}"
20
+ gem "pg", "~> 0.9"
21
+ gem "sqlite3", "~> #{sqlite3_version}"
22
+ gem "seamless_database_pool", "~> 1.0.20"
23
+ end
24
+
25
+ platforms :jruby do
26
+ gem "jdbc-mysql"
27
+ gem "jdbc-postgres"
28
+ gem "activerecord-jdbcsqlite3-adapter", "~> 1.3"
29
+ gem "activerecord-jdbcmysql-adapter", "~> 1.3"
30
+ gem "activerecord-jdbcpostgresql-adapter", "~> 1.3"
31
+ end
32
+
33
+ # Support libs
34
+ gem "factory_bot"
35
+ gem "timecop"
36
+ gem "chronic"
37
+ gem "mocha", "~> 1.3.0"
38
+
39
+ # Debugging
40
+ platforms :jruby do
41
+ gem "ruby-debug", "= 0.10.4"
42
+ end
43
+
44
+ platforms :mri_19 do
45
+ gem "debugger"
46
+ end
47
+
48
+ platforms :ruby do
49
+ gem "pry-byebug"
50
+ gem "rb-readline"
51
+ end
52
+
53
+ if version >= 4.0
54
+ gem "minitest"
55
+ else
56
+ gem "test-unit"
57
+ end
58
+
59
+ eval_gemfile File.expand_path("../gemfiles/#{version}.gemfile", __FILE__)
data/LICENSE ADDED
@@ -0,0 +1,56 @@
1
+ Ruby is copyrighted free software by Yukihiro Matsumoto <matz@netlab.jp>.
2
+ You can redistribute it and/or modify it under either the terms of the
3
+ 2-clause BSDL (see the file BSDL), or the conditions below:
4
+
5
+ 1. You may make and give away verbatim copies of the source form of the
6
+ software without restriction, provided that you duplicate all of the
7
+ original copyright notices and associated disclaimers.
8
+
9
+ 2. You may modify your copy of the software in any way, provided that
10
+ you do at least ONE of the following:
11
+
12
+ a) place your modifications in the Public Domain or otherwise
13
+ make them Freely Available, such as by posting said
14
+ modifications to Usenet or an equivalent medium, or by allowing
15
+ the author to include your modifications in the software.
16
+
17
+ b) use the modified software only within your corporation or
18
+ organization.
19
+
20
+ c) give non-standard binaries non-standard names, with
21
+ instructions on where to get the original software distribution.
22
+
23
+ d) make other distribution arrangements with the author.
24
+
25
+ 3. You may distribute the software in object code or binary form,
26
+ provided that you do at least ONE of the following:
27
+
28
+ a) distribute the binaries and library files of the software,
29
+ together with instructions (in the manual page or equivalent)
30
+ on where to get the original distribution.
31
+
32
+ b) accompany the distribution with the machine-readable source of
33
+ the software.
34
+
35
+ c) give non-standard binaries non-standard names, with
36
+ instructions on where to get the original software distribution.
37
+
38
+ d) make other distribution arrangements with the author.
39
+
40
+ 4. You may modify and include the part of the software into any other
41
+ software (possibly commercial). But some files in the distribution
42
+ are not written by the author, so that they are not under these terms.
43
+
44
+ For the list of those files and their copying conditions, see the
45
+ file LEGAL.
46
+
47
+ 5. The scripts and library files supplied as input to or produced as
48
+ output from the software do not automatically fall under the
49
+ copyright of the software, but belong to whomever generated them,
50
+ and may be sold commercially, and may be aggregated with this
51
+ software.
52
+
53
+ 6. THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
54
+ IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
55
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
56
+ PURPOSE.
@@ -0,0 +1,619 @@
1
+ # Activerecord-Import [![Build Status](https://travis-ci.org/zdennis/activerecord-import.svg?branch=master)](https://travis-ci.org/zdennis/activerecord-import)
2
+
3
+ Activerecord-Import is a library for bulk inserting data using ActiveRecord.
4
+
5
+ One of its major features is following activerecord associations and generating the minimal
6
+ number of SQL insert statements required, avoiding the N+1 insert problem. An example probably
7
+ explains it best. Say you had a schema like this:
8
+
9
+ - Publishers have Books
10
+ - Books have Reviews
11
+
12
+ and you wanted to bulk insert 100 new publishers with 10K books and 3 reviews per book. This library will follow the associations
13
+ down and generate only 3 SQL insert statements - one for the publishers, one for the books, and one for the reviews.
14
+
15
+ In contrast, the standard ActiveRecord save would generate
16
+ 100 insert statements for the publishers, then it would visit each publisher and save all the books:
17
+ 100 * 10,000 = 1,000,000 SQL insert statements
18
+ and then the reviews:
19
+ 100 * 10,000 * 3 = 3M SQL insert statements,
20
+
21
+ That would be about 4M SQL insert statements vs 3, which results in vastly improved performance. In our case, it converted
22
+ an 18 hour batch process to <2 hrs.
23
+
24
+ The gem provides the following high-level features:
25
+
26
+ * Works with raw columns and arrays of values (fastest)
27
+ * Works with model objects (faster)
28
+ * Performs validations (fast)
29
+ * Performs on duplicate key updates (requires MySQL, SQLite 3.24.0+, or Postgres 9.5+)
30
+
31
+ ## Table of Contents
32
+
33
+ * [Examples](#examples)
34
+ * [Introduction](#introduction)
35
+ * [Columns and Arrays](#columns-and-arrays)
36
+ * [Hashes](#hashes)
37
+ * [ActiveRecord Models](#activerecord-models)
38
+ * [Batching](#batching)
39
+ * [Recursive](#recursive)
40
+ * [Options](#options)
41
+ * [Duplicate Key Ignore](#duplicate-key-ignore)
42
+ * [Duplicate Key Update](#duplicate-key-update)
43
+ * [Return Info](#return-info)
44
+ * [Counter Cache](#counter-cache)
45
+ * [ActiveRecord Timestamps](#activerecord-timestamps)
46
+ * [Callbacks](#callbacks)
47
+ * [Supported Adapters](#supported-adapters)
48
+ * [Additional Adapters](#additional-adapters)
49
+ * [Requiring](#requiring)
50
+ * [Autoloading via Bundler](#autoloading-via-bundler)
51
+ * [Manually Loading](#manually-loading)
52
+ * [Load Path Setup](#load-path-setup)
53
+ * [Conflicts With Other Gems](#conflicts-with-other-gems)
54
+ * [More Information](#more-information)
55
+ * [Contributing](#contributing)
56
+ * [Running Tests](#running-tests)
57
+ * [Issue Triage](#issue-triage)
58
+
59
+ ### Examples
60
+
61
+ #### Introduction
62
+
63
+ Without `activerecord-import`, you'd write something like this:
64
+
65
+ ```ruby
66
+ 10.times do |i|
67
+ Book.create! name: "book #{i}"
68
+ end
69
+ ```
70
+
71
+ This would end up making 10 SQL calls. YUCK! With `activerecord-import`, you can instead do this:
72
+
73
+ ```ruby
74
+ books = []
75
+ 10.times do |i|
76
+ books << Book.new(name: "book #{i}")
77
+ end
78
+ Book.import books # or use import!
79
+ ```
80
+
81
+ and only have 1 SQL call. Much better!
82
+
83
+ #### Columns and Arrays
84
+
85
+ The `import` method can take an array of column names (string or symbols) and an array of arrays. Each child array represents an individual record and its list of values in the same order as the columns. This is the fastest import mechanism and also the most primitive.
86
+
87
+ ```ruby
88
+ columns = [ :title, :author ]
89
+ values = [ ['Book1', 'George Orwell'], ['Book2', 'Bob Jones'] ]
90
+
91
+ # Importing without model validations
92
+ Book.import columns, values, validate: false
93
+
94
+ # Import with model validations
95
+ Book.import columns, values, validate: true
96
+
97
+ # when not specified :validate defaults to true
98
+ Book.import columns, values
99
+ ```
100
+
101
+ #### Hashes
102
+
103
+ The `import` method can take an array of hashes. The keys map to the column names in the database.
104
+
105
+ ```ruby
106
+ values = [{ title: 'Book1', author: 'George Orwell' }, { title: 'Book2', author: 'Bob Jones'}]
107
+
108
+ # Importing without model validations
109
+ Book.import values, validate: false
110
+
111
+ # Import with model validations
112
+ Book.import values, validate: true
113
+
114
+ # when not specified :validate defaults to true
115
+ Book.import values
116
+ ```
117
+ #### Import Using Hashes and Explicit Column Names
118
+
119
+ The `import` method can take an array of column names and an array of hash objects. The column names are used to determine what fields of data should be imported. The following example will only import books with the `title` field:
120
+
121
+ ```ruby
122
+ books = [
123
+ { title: "Book 1", author: "George Orwell" },
124
+ { title: "Book 2", author: "Bob Jones" }
125
+ ]
126
+ columns = [ :title ]
127
+
128
+ # without validations
129
+ Book.import columns, books, validate: false
130
+
131
+ # with validations
132
+ Book.import columns, books, validate: true
133
+
134
+ # when not specified :validate defaults to true
135
+ Book.import columns, books
136
+
137
+ # result in table books
138
+ # title | author
139
+ #--------|--------
140
+ # Book 1 | NULL
141
+ # Book 2 | NULL
142
+
143
+ ```
144
+
145
+ Using hashes will only work if the columns are consistent in every hash of the array. If this does not hold, an exception will be raised. There are two workarounds: use the array to instantiate an array of ActiveRecord objects and then pass that into `import` or divide the array into multiple ones with consistent columns and import each one separately.
146
+
147
+ See https://github.com/zdennis/activerecord-import/issues/507 for discussion.
148
+
149
+ ```ruby
150
+ arr = [
151
+ { bar: 'abc' },
152
+ { baz: 'xyz' },
153
+ { bar: '123', baz: '456' }
154
+ ]
155
+
156
+ # An exception will be raised
157
+ Foo.import arr
158
+
159
+ # better
160
+ arr.map! { |args| Foo.new(args) }
161
+ Foo.import arr
162
+
163
+ # better
164
+ arr.group_by(&:keys).each_value do |v|
165
+ Foo.import v
166
+ end
167
+ ```
168
+
169
+ #### ActiveRecord Models
170
+
171
+ The `import` method can take an array of models. The attributes will be pulled off from each model by looking at the columns available on the model.
172
+
173
+ ```ruby
174
+ books = [
175
+ Book.new(title: "Book 1", author: "George Orwell"),
176
+ Book.new(title: "Book 2", author: "Bob Jones")
177
+ ]
178
+
179
+ # without validations
180
+ Book.import books, validate: false
181
+
182
+ # with validations
183
+ Book.import books, validate: true
184
+
185
+ # when not specified :validate defaults to true
186
+ Book.import books
187
+ ```
188
+
189
+ The `import` method can take an array of column names and an array of models. The column names are used to determine what fields of data should be imported. The following example will only import books with the `title` field:
190
+
191
+ ```ruby
192
+ books = [
193
+ Book.new(title: "Book 1", author: "George Orwell"),
194
+ Book.new(title: "Book 2", author: "Bob Jones")
195
+ ]
196
+ columns = [ :title ]
197
+
198
+ # without validations
199
+ Book.import columns, books, validate: false
200
+
201
+ # with validations
202
+ Book.import columns, books, validate: true
203
+
204
+ # when not specified :validate defaults to true
205
+ Book.import columns, books
206
+
207
+ # result in table books
208
+ # title | author
209
+ #--------|--------
210
+ # Book 1 | NULL
211
+ # Book 2 | NULL
212
+
213
+ ```
214
+
215
+ #### Batching
216
+
217
+ The `import` method can take a `batch_size` option to control the number of rows to insert per INSERT statement. The default is the total number of records being inserted so there is a single INSERT statement.
218
+
219
+ ```ruby
220
+ books = [
221
+ Book.new(title: "Book 1", author: "George Orwell"),
222
+ Book.new(title: "Book 2", author: "Bob Jones"),
223
+ Book.new(title: "Book 1", author: "John Doe"),
224
+ Book.new(title: "Book 2", author: "Richard Wright")
225
+ ]
226
+ columns = [ :title ]
227
+
228
+ # 2 INSERT statements for 4 records
229
+ Book.import columns, books, batch_size: 2
230
+ ```
231
+
232
+ #### Recursive
233
+
234
+ NOTE: This only works with PostgreSQL.
235
+
236
+ Assume that Books <code>has_many</code> Reviews.
237
+
238
+ ```ruby
239
+ books = []
240
+ 10.times do |i|
241
+ book = Book.new(name: "book #{i}")
242
+ book.reviews.build(title: "Excellent")
243
+ books << book
244
+ end
245
+ Book.import books, recursive: true
246
+ ```
247
+
248
+ ### Options
249
+
250
+ Key | Options | Default | Description
251
+ ----------------------- | --------------------- | ------------------ | -----------
252
+ :validate | `true`/`false` | `true` | Whether or not to run `ActiveRecord` validations (uniqueness skipped). This option will always be true when using `import!`.
253
+ :validate_uniqueness | `true`/`false` | `false` | Whether or not to run uniqueness validations, has potential pitfalls, use with caution (requires `>= v0.27.0`).
254
+ :on_duplicate_key_ignore| `true`/`false` | `false` | Allows skipping records with duplicate keys. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-ignore) for more details.
255
+ :ignore | `true`/`false` | `false` | Alias for :on_duplicate_key_ignore.
256
+ :on_duplicate_key_update| :all, `Array`, `Hash` | N/A | Allows upsert logic to be used. See [here](https://github.com/zdennis/activerecord-import/#duplicate-key-update) for more details.
257
+ :synchronize | `Array` | N/A | An array of ActiveRecord instances. This synchronizes existing instances in memory with updates from the import.
258
+ :timestamps | `true`/`false` | `true` | Enables/disables timestamps on imported records.
259
+ :recursive | `true`/`false` | `false` | Imports has_many/has_one associations (PostgreSQL only).
260
+ :batch_size | `Integer` | total # of records | Max number of records to insert per import
261
+ :raise_error | `true`/`false` | `false` | Throws an exception if there are invalid records. `import!` is a shortcut for this.
262
+
263
+
264
+ #### Duplicate Key Ignore
265
+
266
+ [MySQL](http://dev.mysql.com/doc/refman/5.0/en/insert-on-duplicate.html), [SQLite](https://www.sqlite.org/lang_insert.html), and [PostgreSQL](https://www.postgresql.org/docs/current/static/sql-insert.html#SQL-ON-CONFLICT) (9.5+) support `on_duplicate_key_ignore` which allows you to skip records if a primary or unique key constraint is violated.
267
+
268
+ For Postgres 9.5+ it adds `ON CONFLICT DO NOTHING`, for MySQL it uses `INSERT IGNORE`, and for SQLite it uses `INSERT OR IGNORE`. Cannot be enabled on a recursive import. For database adapters that normally support setting primary keys on imported objects, this option prevents that from occurring.
269
+
270
+ ```ruby
271
+ book = Book.create! title: "Book1", author: "George Orwell"
272
+ book.title = "Updated Book Title"
273
+ book.author = "Bob Barker"
274
+
275
+ Book.import [book], on_duplicate_key_ignore: true
276
+
277
+ book.reload.title # => "Book1" (stayed the same)
278
+ book.reload.author # => "George Orwell" (stayed the same)
279
+ ```
280
+
281
+ The option `:on_duplicate_key_ignore` is bypassed when `:recursive` is enabled for [PostgreSQL imports](https://github.com/zdennis/activerecord-import/wiki#recursive-example-postgresql-only).
282
+
283
+ #### Duplicate Key Update
284
+
285
+ MySQL, PostgreSQL (9.5+), and SQLite (3.24.0+) support `on duplicate key update` (also known as "upsert") which allows you to specify fields whose values should be updated if a primary or unique key constraint is violated.
286
+
287
+ One big difference between MySQL and PostgreSQL support is that MySQL will handle any conflict that happens, but PostgreSQL requires that you specify which columns the conflict would occur over. SQLite models its upsert support after PostgreSQL.
288
+
289
+ This will use MySQL's `ON DUPLICATE KEY UPDATE` or Postgres/SQLite `ON CONFLICT DO UPDATE` to do upsert.
290
+
291
+ Basic Update
292
+
293
+ ```ruby
294
+ book = Book.create! title: "Book1", author: "George Orwell"
295
+ book.title = "Updated Book Title"
296
+ book.author = "Bob Barker"
297
+
298
+ # MySQL version
299
+ Book.import [book], on_duplicate_key_update: [:title]
300
+
301
+ # PostgreSQL version
302
+ Book.import [book], on_duplicate_key_update: {conflict_target: [:id], columns: [:title]}
303
+
304
+ # PostgreSQL shorthand version (conflict target must be primary key)
305
+ Book.import [book], on_duplicate_key_update: [:title]
306
+
307
+ book.reload.title # => "Updated Book Title" (changed)
308
+ book.reload.author # => "George Orwell" (stayed the same)
309
+ ```
310
+
311
+ Using the value from another column
312
+
313
+ ```ruby
314
+ book = Book.create! title: "Book1", author: "George Orwell"
315
+ book.title = "Updated Book Title"
316
+
317
+ # MySQL version
318
+ Book.import [book], on_duplicate_key_update: {author: :title}
319
+
320
+ # PostgreSQL version (no shorthand version)
321
+ Book.import [book], on_duplicate_key_update: {
322
+ conflict_target: [:id], columns: {author: :title}
323
+ }
324
+
325
+ book.reload.title # => "Book1" (stayed the same)
326
+ book.reload.author # => "Updated Book Title" (changed)
327
+ ```
328
+
329
+ Using Custom SQL
330
+
331
+ ```ruby
332
+ book = Book.create! title: "Book1", author: "George Orwell"
333
+ book.author = "Bob Barker"
334
+
335
+ # MySQL version
336
+ Book.import [book], on_duplicate_key_update: "author = values(author)"
337
+
338
+ # PostgreSQL version
339
+ Book.import [book], on_duplicate_key_update: {
340
+ conflict_target: [:id], columns: "author = excluded.author"
341
+ }
342
+
343
+ # PostgreSQL shorthand version (conflict target must be primary key)
344
+ Book.import [book], on_duplicate_key_update: "author = excluded.author"
345
+
346
+ book.reload.title # => "Book1" (stayed the same)
347
+ book.reload.author # => "Bob Barker" (changed)
348
+ ```
349
+
350
+ PostgreSQL Using constraints
351
+
352
+ ```ruby
353
+ book = Book.create! title: "Book1", author: "George Orwell", edition: 3, published_at: nil
354
+ book.published_at = Time.now
355
+
356
+ # in migration
357
+ execute <<-SQL
358
+ ALTER TABLE books
359
+ ADD CONSTRAINT for_upsert UNIQUE (title, author, edition);
360
+ SQL
361
+
362
+ # PostgreSQL version
363
+ Book.import [book], on_duplicate_key_update: {constraint_name: :for_upsert, columns: [:published_at]}
364
+
365
+
366
+ book.reload.title # => "Book1" (stayed the same)
367
+ book.reload.author # => "George Orwell" (stayed the same)
368
+ book.reload.edition # => 3 (stayed the same)
369
+ book.reload.published_at # => 2017-10-09 (changed)
370
+ ```
371
+
372
+ ```ruby
373
+ Book.import books, validate_uniqueness: true
374
+ ```
375
+
376
+ ### Return Info
377
+
378
+ The `import` method returns a `Result` object that responds to `failed_instances` and `num_inserts`. Additionally, for users of Postgres, there will be two arrays `ids` and `results` that can be accessed`.
379
+
380
+ ```ruby
381
+ articles = [
382
+ Article.new(author_id: 1, title: 'First Article', content: 'This is the first article'),
383
+ Article.new(author_id: 2, title: 'Second Article', content: ''),
384
+ Article.new(author_id: 3, content: '')
385
+ ]
386
+
387
+ demo = Article.import(articles, returning: :title) # => #<struct ActiveRecord::Import::Result
388
+
389
+ demo.failed_instances
390
+ => [#<Article id: 3, author_id: 3, title: nil, content: "", created_at: nil, updated_at: nil>]
391
+
392
+ demo.num_inserts
393
+ => 1,
394
+
395
+ demo.ids
396
+ => ["1", "2"] # for Postgres
397
+ => [] # for other DBs
398
+
399
+ demo.results
400
+ => ["First Article", "Second Article"] # for Postgres
401
+ => [] # for other DBs
402
+ ```
403
+
404
+ ### Counter Cache
405
+
406
+ When running `import`, `activerecord-import` does not automatically update counter cache columns. To update these columns, you will need to do one of the following:
407
+
408
+ * Provide values to the column as an argument on your object that is passed in.
409
+ * Manually update the column after the record has been imported.
410
+
411
+ ### ActiveRecord Timestamps
412
+
413
+ If you're familiar with ActiveRecord you're probably familiar with its timestamp columns: created_at, created_on, updated_at, updated_on, etc. When importing data the timestamp fields will continue to work as expected and each timestamp column will be set.
414
+
415
+ Should you wish to specify those columns, you may use the option `timestamps: false`.
416
+
417
+ However, it is also possible to set just `:created_at` in specific records. In this case despite using `timestamps: true`, `:created_at` will be updated only in records where that field is `nil`. Same rule applies for record associations when enabling the option `recursive: true`.
418
+
419
+ If you are using custom time zones, these will be respected when performing imports as well as long as `ActiveRecord::Base.default_timezone` is set, which for practically all Rails apps it is
420
+
421
+ ### Callbacks
422
+
423
+ ActiveRecord callbacks related to [creating](http://guides.rubyonrails.org/active_record_callbacks.html#creating-an-object), [updating](http://guides.rubyonrails.org/active_record_callbacks.html#updating-an-object), or [destroying](http://guides.rubyonrails.org/active_record_callbacks.html#destroying-an-object) records (other than `before_validation` and `after_validation`) will NOT be called when calling the import method. This is because it is mass importing rows of data and doesn't necessarily have access to in-memory ActiveRecord objects.
424
+
425
+ If you do have a collection of in-memory ActiveRecord objects you can do something like this:
426
+
427
+ ```ruby
428
+ books.each do |book|
429
+ book.run_callbacks(:save) { false }
430
+ book.run_callbacks(:create) { false }
431
+ end
432
+ Book.import(books)
433
+ ```
434
+
435
+ This will run before_create and before_save callbacks on each item. The `false` argument is needed to prevent after_save being run, which wouldn't make sense prior to bulk import. Something to note in this example is that the before_create and before_save callbacks will run before the validation callbacks.
436
+
437
+ If that is an issue, another possible approach is to loop through your models first to do validations and then only run callbacks on and import the valid models.
438
+
439
+ ```ruby
440
+ valid_books = []
441
+ invalid_books = []
442
+
443
+ books.each do |book|
444
+ if book.valid?
445
+ valid_books << book
446
+ else
447
+ invalid_books << book
448
+ end
449
+ end
450
+
451
+ valid_books.each do |book|
452
+ book.run_callbacks(:save) { false }
453
+ book.run_callbacks(:create) { false }
454
+ end
455
+
456
+ Book.import valid_books, validate: false
457
+ ```
458
+
459
+ ### Supported Adapters
460
+
461
+ The following database adapters are currently supported:
462
+
463
+ * MySQL - supports core import functionality plus on duplicate key update support (included in activerecord-import 0.1.0 and higher)
464
+ * MySQL2 - supports core import functionality plus on duplicate key update support (included in activerecord-import 0.2.0 and higher)
465
+ * PostgreSQL - supports core import functionality (included in activerecord-import 0.1.0 and higher)
466
+ * SQLite3 - supports core import functionality (included in activerecord-import 0.1.0 and higher)
467
+ * Oracle - supports core import functionality through DML trigger (available as an external gem: [activerecord-import-oracle_enhanced](https://github.com/keeguon/activerecord-import-oracle_enhanced)
468
+ * SQL Server - supports core import functionality (available as an external gem: [activerecord-import-sqlserver](https://github.com/keeguon/activerecord-import-sqlserver)
469
+
470
+ If your adapter isn't listed here, please consider creating an external gem as described in the README to provide support. If you do, feel free to update this wiki to include a link to the new adapter's repository!
471
+
472
+ To test which features are supported by your adapter, use the following methods on a model class:
473
+ * `supports_import?(*args)`
474
+ * `supports_on_duplicate_key_update?`
475
+ * `supports_setting_primary_key_of_imported_objects?`
476
+
477
+ ### Additional Adapters
478
+
479
+ Additional adapters can be provided by gems external to activerecord-import by providing an adapter that matches the naming convention setup by activerecord-import (and subsequently activerecord) for dynamically loading adapters. This involves also providing a folder on the load path that follows the activerecord-import naming convention to allow activerecord-import to dynamically load the file.
480
+
481
+ When `ActiveRecord::Import.require_adapter("fake_name")` is called the require will be:
482
+
483
+ ```ruby
484
+ require 'activerecord-import/active_record/adapters/fake_name_adapter'
485
+ ```
486
+
487
+ This allows an external gem to dynamically add an adapter without the need to add any file/code to the core activerecord-import gem.
488
+
489
+ ### Requiring
490
+
491
+ Note: These instructions will only work if you are using version 0.2.0 or higher.
492
+
493
+ #### Autoloading via Bundler
494
+
495
+ If you are using Rails or otherwise autoload your dependencies via Bundler, all you need to do add the gem to your `Gemfile` like so:
496
+
497
+ ```ruby
498
+ gem 'activerecord-import'
499
+ ```
500
+
501
+ #### Manually Loading
502
+
503
+ You may want to manually load activerecord-import for one reason or another. First, add the `require: false` argument like so:
504
+
505
+ ```ruby
506
+ gem 'activerecord-import', require: false
507
+ ```
508
+
509
+ This will allow you to load up activerecord-import in the file or files where you are using it and only load the parts you need.
510
+ If you are doing this within Rails and ActiveRecord has established a database connection (such as within a controller), you will need to do extra initialization work:
511
+
512
+ ```ruby
513
+ require 'activerecord-import/base'
514
+ # load the appropriate database adapter (postgresql, mysql2, sqlite3, etc)
515
+ require 'activerecord-import/active_record/adapters/postgresql_adapter'
516
+ ```
517
+
518
+ If your gem dependencies aren’t autoloaded, and your script will be establishing a database connection, then simply require activerecord-import after ActiveRecord has been loaded, i.e.:
519
+
520
+ ```ruby
521
+ require 'active_record'
522
+ require 'activerecord-import'
523
+ ```
524
+
525
+ ### Load Path Setup
526
+ To understand how rubygems loads code you can reference the following:
527
+
528
+ http://guides.rubygems.org/patterns/#loading_code
529
+
530
+ And an example of how active_record dynamically load adapters:
531
+
532
+ https://github.com/rails/rails/blob/master/activerecord/lib/active_record/connection_adapters/connection_specification.rb
533
+
534
+ In summary, when a gem is loaded rubygems adds the `lib` folder of the gem to the global load path `$LOAD_PATH` so that all `require` lookups will not propagate through all of the folders on the load path. When a `require` is issued each folder on the `$LOAD_PATH` is checked for the file and/or folder referenced. This allows a gem (like activerecord-import) to define push the activerecord-import folder (or namespace) on the `$LOAD_PATH` and any adapters provided by activerecord-import will be found by rubygems when the require is issued.
535
+
536
+ If `fake_name` adapter is needed by a gem (potentially called `activerecord-import-fake_name`) then the folder structure should look as follows:
537
+
538
+ ```bash
539
+ activerecord-import-fake_name/
540
+ |-- activerecord-import-fake_name.gemspec
541
+ |-- lib
542
+ | |-- activerecord-import-fake_name.rb
543
+ | |-- activerecord-import-fake_name
544
+ | | |-- version.rb
545
+ | |-- activerecord-import
546
+ | | |-- active_record
547
+ | | | |-- adapters
548
+ | | | |-- fake_name_adapter.rb
549
+ ```
550
+
551
+ When rubygems pushes the `lib` folder onto the load path a `require` will now find `activerecord-import/active_record/adapters/fake_name_adapter` as it runs through the lookup process for a ruby file under that path in `$LOAD_PATH`
552
+
553
+
554
+ ### Conflicts With Other Gems
555
+
556
+ Activerecord-Import adds the `.import` method onto `ActiveRecord::Base`. There are other gems, such as `elasticsearch-rails`, that do the same thing. In conflicts such as this, there is an aliased method named `.bulk_import` that can be used interchangeably.
557
+
558
+ If you are using the `apartment` gem, there is a weird triple interaction between that gem, `activerecord-import`, and `activerecord` involving caching of the `sequence_name` of a model. This can be worked around by explcitly setting this value within the model. For example:
559
+
560
+ ```ruby
561
+ class Post < ActiveRecord::Base
562
+ self.sequence_name = "posts_seq"
563
+ end
564
+ ```
565
+
566
+ Another way to work around the issue is to call `.reset_sequence_name` on the model. For example:
567
+
568
+ ```ruby
569
+ schemas.all.each do |schema|
570
+ Apartment::Tenant.switch! schema.name
571
+ ActiveRecord::Base.transaction do
572
+ Post.reset_sequence_name
573
+
574
+ Post.import posts
575
+ end
576
+ end
577
+ ```
578
+
579
+ See https://github.com/zdennis/activerecord-import/issues/233 for further discussion.
580
+
581
+ ### More Information
582
+
583
+ For more information on Activerecord-Import please see its wiki: https://github.com/zdennis/activerecord-import/wiki
584
+
585
+ To document new information, please add to the README instead of the wiki. See https://github.com/zdennis/activerecord-import/issues/397 for discussion.
586
+
587
+ ### Contributing
588
+
589
+ #### Running Tests
590
+
591
+ The first thing you need to do is set up your database(s):
592
+
593
+ * copy `test/database.yml.sample` to `test/database.yml`
594
+ * modify `test/database.yml` for your database settings
595
+ * create databases as needed
596
+
597
+ After that, you can run the tests. They run against multiple tests and ActiveRecord versions.
598
+
599
+ This is one example of how to run the tests:
600
+
601
+ ```ruby
602
+ rm Gemfile.lock
603
+ AR_VERSION=4.2 bundle install
604
+ AR_VERSION=4.2 bundle exec rake test:postgresql test:sqlite3 test:mysql2
605
+ ```
606
+
607
+ Once you have pushed up your changes, you can find your CI results [here](https://travis-ci.org/zdennis/activerecord-import/).
608
+
609
+ ## Issue Triage [![Open Source Helpers](https://www.codetriage.com/zdennis/activerecord-import/badges/users.svg)](https://www.codetriage.com/zdennis/activerecord-import)
610
+
611
+ You can triage issues which may include reproducing bug reports or asking for vital information, such as version numbers or reproduction instructions. If you would like to start triaging issues, one easy way to get started is to [subscribe to activerecord-import on CodeTriage](https://www.codetriage.com/zdennis/activerecord-import).
612
+
613
+ # License
614
+
615
+ This is licensed under the ruby license.
616
+
617
+ # Author
618
+
619
+ Zach Dennis (zach.dennis@gmail.com)