bulk_insert 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +11 -6
- data/lib/bulk_insert/statement_adapters.rb +22 -0
- data/lib/bulk_insert/statement_adapters/base_adapter.rb +21 -0
- data/lib/bulk_insert/statement_adapters/generic_adapter.rb +19 -0
- data/lib/bulk_insert/statement_adapters/mysql_adapter.rb +24 -0
- data/lib/bulk_insert/statement_adapters/postgresql_adapter.rb +26 -0
- data/lib/bulk_insert/statement_adapters/sqlite_adapter.rb +19 -0
- data/lib/bulk_insert/version.rb +1 -1
- data/lib/bulk_insert/worker.rb +7 -36
- data/test/bulk_insert/worker_test.rb +118 -83
- metadata +35 -36
- data/test/dummy/db/development.sqlite3 +0 -0
- data/test/dummy/db/test.sqlite3 +0 -0
- data/test/dummy/log/development.log +0 -17
- data/test/dummy/log/test.log +0 -9034
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 7f92126ff36c331b40bfc5ca7d261af01410963a5c9dab65824a64f36405e102
|
4
|
+
data.tar.gz: ef41beaeb0f13ff251818e0a57c0e729f2a3bf520d536a24182c2213e0109262
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e6f50b3cd6d20bc348c0cf6c94c3e920a54e26c5b1ba2019209b2521febdffcc4fdf9a3bf39d688990a0e3907fbbf244d3ed53e3ecbe53db283ea3aec7342dbf
|
7
|
+
data.tar.gz: 1849fab8452c635004419e67e8748428e142875aa0c4b4920d4daecab66ed1a2b6ca863124704ae4d847e5c4fce5d38ec8911cb95cc5aad0f533682bc869bc65
|
data/README.md
CHANGED
@@ -104,7 +104,6 @@ empty the batch so that you can add more rows to it if you want. Note
|
|
104
104
|
that all records saved together will have the same created_at/updated_at
|
105
105
|
timestamp (unless one was explicitly set).
|
106
106
|
|
107
|
-
|
108
107
|
### Batch Set Size
|
109
108
|
|
110
109
|
By default, the size of the insert is limited to 500 rows at a time.
|
@@ -149,22 +148,29 @@ Book.bulk_insert(*destination_columns, ignore: true) do |worker|
|
|
149
148
|
end
|
150
149
|
```
|
151
150
|
|
152
|
-
### Update Duplicates (MySQL)
|
151
|
+
### Update Duplicates (MySQL, PostgreSQL)
|
153
152
|
|
154
153
|
If you don't want to ignore duplicate rows but instead want to update them
|
155
154
|
then you can use the _update_duplicates_ option. Set this option to true
|
156
|
-
|
157
|
-
|
155
|
+
(MySQL) or list unique column names (PostgreSQL) and when a duplicate row
|
156
|
+
is found the row will be updated with your new values.
|
157
|
+
Default value for this option is false.
|
158
158
|
|
159
159
|
```ruby
|
160
160
|
destination_columns = [:title, :author]
|
161
161
|
|
162
|
-
# Update duplicate rows
|
162
|
+
# Update duplicate rows (MySQL)
|
163
163
|
Book.bulk_insert(*destination_columns, update_duplicates: true) do |worker|
|
164
164
|
worker.add(...)
|
165
165
|
worker.add(...)
|
166
166
|
# ...
|
167
167
|
end
|
168
|
+
|
169
|
+
# Update duplicate rows (PostgreSQL)
|
170
|
+
Book.bulk_insert(*destination_columns, update_duplicates: %w[title]) do |worker|
|
171
|
+
worker.add(...)
|
172
|
+
# ...
|
173
|
+
end
|
168
174
|
```
|
169
175
|
|
170
176
|
### Return Primary Keys (PostgreSQL, PostGIS)
|
@@ -185,7 +191,6 @@ end
|
|
185
191
|
worker.result_sets
|
186
192
|
```
|
187
193
|
|
188
|
-
|
189
194
|
## License
|
190
195
|
|
191
196
|
BulkInsert is released under the MIT license (see MIT-LICENSE) by
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require_relative 'statement_adapters/generic_adapter'
|
2
|
+
require_relative 'statement_adapters/mysql_adapter'
|
3
|
+
require_relative 'statement_adapters/postgresql_adapter'
|
4
|
+
require_relative 'statement_adapters/sqlite_adapter'
|
5
|
+
|
6
|
+
module BulkInsert
|
7
|
+
module StatementAdapters
|
8
|
+
def adapter_for(connection)
|
9
|
+
case connection.adapter_name
|
10
|
+
when /^mysql/i
|
11
|
+
MySQLAdapter.new
|
12
|
+
when /\APost(?:greSQL|GIS)/i
|
13
|
+
PostgreSQLAdapter.new
|
14
|
+
when /\ASQLite/i
|
15
|
+
SQLiteAdapter.new
|
16
|
+
else
|
17
|
+
GenericAdapter.new
|
18
|
+
end
|
19
|
+
end
|
20
|
+
module_function :adapter_for
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module BulkInsert
|
2
|
+
module StatementAdapters
|
3
|
+
class BaseAdapter
|
4
|
+
def initialize
|
5
|
+
raise "You cannot initialize base adapter" if self.class == BaseAdapter
|
6
|
+
end
|
7
|
+
|
8
|
+
def insert_ignore_statement
|
9
|
+
raise "Not implemented"
|
10
|
+
end
|
11
|
+
|
12
|
+
def on_conflict_statement(_columns, _ignore, _update_duplicates)
|
13
|
+
raise "Not implemented"
|
14
|
+
end
|
15
|
+
|
16
|
+
def primary_key_return_statement(_primary_key)
|
17
|
+
raise "Not implemented"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require_relative 'base_adapter'
|
2
|
+
|
3
|
+
module BulkInsert
|
4
|
+
module StatementAdapters
|
5
|
+
class GenericAdapter < BaseAdapter
|
6
|
+
def insert_ignore_statement
|
7
|
+
''
|
8
|
+
end
|
9
|
+
|
10
|
+
def on_conflict_statement(_columns, _ignore, _update_duplicates)
|
11
|
+
''
|
12
|
+
end
|
13
|
+
|
14
|
+
def primary_key_return_statement(_primary_key)
|
15
|
+
''
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require_relative 'base_adapter'
|
2
|
+
|
3
|
+
module BulkInsert
|
4
|
+
module StatementAdapters
|
5
|
+
class MySQLAdapter < BaseAdapter
|
6
|
+
def insert_ignore_statement
|
7
|
+
'IGNORE'
|
8
|
+
end
|
9
|
+
|
10
|
+
def on_conflict_statement(columns, _ignore, update_duplicates)
|
11
|
+
return '' unless update_duplicates
|
12
|
+
|
13
|
+
update_values = columns.map do |column|
|
14
|
+
"`#{column.name}`=VALUES(`#{column.name}`)"
|
15
|
+
end.join(', ')
|
16
|
+
' ON DUPLICATE KEY UPDATE ' + update_values
|
17
|
+
end
|
18
|
+
|
19
|
+
def primary_key_return_statement(_primary_key)
|
20
|
+
''
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require_relative 'base_adapter'
|
2
|
+
|
3
|
+
module BulkInsert
|
4
|
+
module StatementAdapters
|
5
|
+
class PostgreSQLAdapter < BaseAdapter
|
6
|
+
def insert_ignore_statement
|
7
|
+
''
|
8
|
+
end
|
9
|
+
|
10
|
+
def on_conflict_statement(columns, ignore, update_duplicates)
|
11
|
+
if ignore
|
12
|
+
' ON CONFLICT DO NOTHING'
|
13
|
+
elsif update_duplicates
|
14
|
+
update_values = columns.map do |column|
|
15
|
+
"#{column.name}=EXCLUDED.#{column.name}"
|
16
|
+
end.join(', ')
|
17
|
+
' ON CONFLICT(' + update_duplicates.join(', ') + ') DO UPDATE SET ' + update_values
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def primary_key_return_statement(primary_key)
|
22
|
+
" RETURNING #{primary_key}"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require_relative 'base_adapter'
|
2
|
+
|
3
|
+
module BulkInsert
|
4
|
+
module StatementAdapters
|
5
|
+
class SQLiteAdapter < BaseAdapter
|
6
|
+
def insert_ignore_statement
|
7
|
+
'OR IGNORE'
|
8
|
+
end
|
9
|
+
|
10
|
+
def on_conflict_statement(_columns, _ignore, _update_duplicates)
|
11
|
+
''
|
12
|
+
end
|
13
|
+
|
14
|
+
def primary_key_return_statement(_primary_key)
|
15
|
+
''
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
data/lib/bulk_insert/version.rb
CHANGED
data/lib/bulk_insert/worker.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
require_relative 'statement_adapters'
|
2
|
+
|
1
3
|
module BulkInsert
|
2
4
|
class Worker
|
3
5
|
attr_reader :connection
|
@@ -8,6 +10,8 @@ module BulkInsert
|
|
8
10
|
attr_reader :ignore, :update_duplicates, :result_sets
|
9
11
|
|
10
12
|
def initialize(connection, table_name, primary_key, column_names, set_size=500, ignore=false, update_duplicates=false, return_primary_keys=false)
|
13
|
+
@statement_adapter = StatementAdapters.adapter_for(connection)
|
14
|
+
|
11
15
|
@connection = connection
|
12
16
|
@set_size = set_size
|
13
17
|
|
@@ -116,8 +120,8 @@ module BulkInsert
|
|
116
120
|
|
117
121
|
if !rows.empty?
|
118
122
|
sql << rows.join(",")
|
119
|
-
sql << on_conflict_statement
|
120
|
-
sql << primary_key_return_statement
|
123
|
+
sql << @statement_adapter.on_conflict_statement(@columns, ignore, update_duplicates)
|
124
|
+
sql << @statement_adapter.primary_key_return_statement(@primary_key) if @return_primary_keys
|
121
125
|
sql
|
122
126
|
else
|
123
127
|
false
|
@@ -125,41 +129,8 @@ module BulkInsert
|
|
125
129
|
end
|
126
130
|
|
127
131
|
def insert_sql_statement
|
132
|
+
insert_ignore = @ignore ? @statement_adapter.insert_ignore_statement : ''
|
128
133
|
"INSERT #{insert_ignore} INTO #{@table_name} (#{@column_names}) VALUES "
|
129
134
|
end
|
130
|
-
|
131
|
-
def insert_ignore
|
132
|
-
if ignore
|
133
|
-
case adapter_name
|
134
|
-
when /^mysql/i
|
135
|
-
'IGNORE'
|
136
|
-
when /\ASQLite/i # SQLite
|
137
|
-
'OR IGNORE'
|
138
|
-
else
|
139
|
-
'' # Not supported
|
140
|
-
end
|
141
|
-
end
|
142
|
-
end
|
143
|
-
|
144
|
-
def primary_key_return_statement
|
145
|
-
if @return_primary_keys && adapter_name =~ /\APost(?:greSQL|GIS)/i
|
146
|
-
" RETURNING #{@primary_key}"
|
147
|
-
else
|
148
|
-
''
|
149
|
-
end
|
150
|
-
end
|
151
|
-
|
152
|
-
def on_conflict_statement
|
153
|
-
if (adapter_name =~ /\APost(?:greSQL|GIS)/i && ignore )
|
154
|
-
' ON CONFLICT DO NOTHING'
|
155
|
-
elsif adapter_name =~ /^mysql/i && update_duplicates
|
156
|
-
update_values = @columns.map do |column|
|
157
|
-
"`#{column.name}`=VALUES(`#{column.name}`)"
|
158
|
-
end.join(', ')
|
159
|
-
' ON DUPLICATE KEY UPDATE ' + update_values
|
160
|
-
else
|
161
|
-
''
|
162
|
-
end
|
163
|
-
end
|
164
135
|
end
|
165
136
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'minitest/mock'
|
1
2
|
require 'test_helper'
|
2
3
|
|
3
4
|
class BulkInsertWorkerTest < ActiveSupport::TestCase
|
@@ -270,93 +271,124 @@ class BulkInsertWorkerTest < ActiveSupport::TestCase
|
|
270
271
|
end
|
271
272
|
|
272
273
|
test "adapter dependent mysql methods" do
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
274
|
+
connection = Testing.connection
|
275
|
+
connection.stub :adapter_name, 'MySQL' do
|
276
|
+
mysql_worker = BulkInsert::Worker.new(
|
277
|
+
connection,
|
278
|
+
Testing.table_name,
|
279
|
+
'id',
|
280
|
+
%w(greeting age happy created_at updated_at color),
|
281
|
+
500, # batch size
|
282
|
+
true # ignore
|
283
|
+
)
|
284
|
+
|
285
|
+
assert_equal mysql_worker.adapter_name, 'MySQL'
|
286
|
+
assert_equal (mysql_worker.adapter_name == 'MySQL'), true
|
287
|
+
assert_equal mysql_worker.ignore, true
|
288
|
+
assert_equal ((mysql_worker.adapter_name == 'MySQL') & mysql_worker.ignore), true
|
289
|
+
|
290
|
+
mysql_worker.add ["Yo", 15, false, nil, nil]
|
291
|
+
|
292
|
+
assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')"
|
293
|
+
end
|
290
294
|
end
|
291
295
|
|
292
296
|
test "adapter dependent mysql methods work for mysql2" do
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
297
|
+
connection = Testing.connection
|
298
|
+
connection.stub :adapter_name, 'Mysql2' do
|
299
|
+
mysql_worker = BulkInsert::Worker.new(
|
300
|
+
connection,
|
301
|
+
Testing.table_name,
|
302
|
+
'id',
|
303
|
+
%w(greeting age happy created_at updated_at color),
|
304
|
+
500, # batch size
|
305
|
+
true, # ignore
|
306
|
+
true) # update_duplicates
|
307
|
+
|
308
|
+
assert_equal mysql_worker.adapter_name, 'Mysql2'
|
309
|
+
assert mysql_worker.ignore
|
310
|
+
|
311
|
+
mysql_worker.add ["Yo", 15, false, nil, nil]
|
312
|
+
|
313
|
+
assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)"
|
314
|
+
end
|
309
315
|
end
|
310
316
|
|
311
317
|
test "adapter dependent Mysql2Spatial methods" do
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
318
|
+
connection = Testing.connection
|
319
|
+
connection.stub :adapter_name, 'Mysql2Spatial' do
|
320
|
+
mysql_worker = BulkInsert::Worker.new(
|
321
|
+
connection,
|
322
|
+
Testing.table_name,
|
323
|
+
'id',
|
324
|
+
%w(greeting age happy created_at updated_at color),
|
325
|
+
500, # batch size
|
326
|
+
true) # ignore
|
320
327
|
|
321
|
-
|
328
|
+
assert_equal mysql_worker.adapter_name, 'Mysql2Spatial'
|
322
329
|
|
323
|
-
|
330
|
+
mysql_worker.add ["Yo", 15, false, nil, nil]
|
324
331
|
|
325
|
-
|
332
|
+
assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')"
|
333
|
+
end
|
326
334
|
end
|
327
335
|
|
328
336
|
test "adapter dependent postgresql methods" do
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
337
|
+
connection = Testing.connection
|
338
|
+
connection.stub :adapter_name, 'PostgreSQL' do
|
339
|
+
pgsql_worker = BulkInsert::Worker.new(
|
340
|
+
connection,
|
341
|
+
Testing.table_name,
|
342
|
+
'id',
|
343
|
+
%w(greeting age happy created_at updated_at color),
|
344
|
+
500, # batch size
|
345
|
+
true, # ignore
|
346
|
+
false, # update duplicates
|
347
|
+
true # return primary keys
|
348
|
+
)
|
349
|
+
|
350
|
+
pgsql_worker.add ["Yo", 15, false, nil, nil]
|
351
|
+
|
352
|
+
assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id"
|
353
|
+
end
|
354
|
+
end
|
341
355
|
|
342
|
-
|
356
|
+
test "adapter dependent postgresql methods (with update_duplicates)" do
|
357
|
+
connection = Testing.connection
|
358
|
+
connection.stub :adapter_name, 'PostgreSQL' do
|
359
|
+
pgsql_worker = BulkInsert::Worker.new(
|
360
|
+
connection,
|
361
|
+
Testing.table_name,
|
362
|
+
'id',
|
363
|
+
%w(greeting age happy created_at updated_at color),
|
364
|
+
500, # batch size
|
365
|
+
false, # ignore
|
366
|
+
%w(greeting age happy), # update duplicates
|
367
|
+
true # return primary keys
|
368
|
+
)
|
369
|
+
pgsql_worker.add ["Yo", 15, false, nil, nil]
|
370
|
+
|
371
|
+
assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON CONFLICT(greeting, age, happy) DO UPDATE SET greeting=EXCLUDED.greeting, age=EXCLUDED.age, happy=EXCLUDED.happy, created_at=EXCLUDED.created_at, updated_at=EXCLUDED.updated_at, color=EXCLUDED.color RETURNING id"
|
372
|
+
end
|
343
373
|
end
|
344
374
|
|
345
375
|
test "adapter dependent PostGIS methods" do
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
376
|
+
connection = Testing.connection
|
377
|
+
connection.stub :adapter_name, 'PostGIS' do
|
378
|
+
pgsql_worker = BulkInsert::Worker.new(
|
379
|
+
connection,
|
380
|
+
Testing.table_name,
|
381
|
+
'id',
|
382
|
+
%w(greeting age happy created_at updated_at color),
|
383
|
+
500, # batch size
|
384
|
+
true, # ignore
|
385
|
+
false, # update duplicates
|
386
|
+
true # return primary keys
|
387
|
+
)
|
388
|
+
pgsql_worker.add ["Yo", 15, false, nil, nil]
|
389
|
+
|
390
|
+
assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id"
|
391
|
+
end
|
360
392
|
end
|
361
393
|
|
362
394
|
test "adapter dependent sqlite3 methods (with lowercase adapter name)" do
|
@@ -388,17 +420,20 @@ class BulkInsertWorkerTest < ActiveSupport::TestCase
|
|
388
420
|
end
|
389
421
|
|
390
422
|
test "mysql adapter can update duplicates" do
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
423
|
+
connection = Testing.connection
|
424
|
+
connection.stub :adapter_name, 'MySQL' do
|
425
|
+
mysql_worker = BulkInsert::Worker.new(
|
426
|
+
connection,
|
427
|
+
Testing.table_name,
|
428
|
+
'id',
|
429
|
+
%w(greeting age happy created_at updated_at color),
|
430
|
+
500, # batch size
|
431
|
+
false, # ignore
|
432
|
+
true # update_duplicates
|
433
|
+
)
|
434
|
+
mysql_worker.add ["Yo", 15, false, nil, nil]
|
435
|
+
|
436
|
+
assert_equal mysql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)"
|
437
|
+
end
|
403
438
|
end
|
404
439
|
end
|