bulk_insert 1.7.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 9a76d84f0be15b3ba8e0186439eca425dc9c9d07
4
- data.tar.gz: c9fd5eb614204c838fed55e0ee7bc8075dec0b47
2
+ SHA256:
3
+ metadata.gz: 7f92126ff36c331b40bfc5ca7d261af01410963a5c9dab65824a64f36405e102
4
+ data.tar.gz: ef41beaeb0f13ff251818e0a57c0e729f2a3bf520d536a24182c2213e0109262
5
5
  SHA512:
6
- metadata.gz: 533c701360f76f710a875c40f1a0d6c5d463eb8a64c8b32a9144559e8972b85dbceeedda64fd5b7d74587ebb7bfef426217a6c026aab3c1fe408fe7d96e826b8
7
- data.tar.gz: 00b3dd6b1c0826ca6c3eec7a64d61f609414be09d266c183e212c6dcc9886c34f731cd7a8d8785e592b6a98eee3933bf25c62c0388d387992308917923ccde61
6
+ metadata.gz: e6f50b3cd6d20bc348c0cf6c94c3e920a54e26c5b1ba2019209b2521febdffcc4fdf9a3bf39d688990a0e3907fbbf244d3ed53e3ecbe53db283ea3aec7342dbf
7
+ data.tar.gz: 1849fab8452c635004419e67e8748428e142875aa0c4b4920d4daecab66ed1a2b6ca863124704ae4d847e5c4fce5d38ec8911cb95cc5aad0f533682bc869bc65
data/README.md CHANGED
@@ -104,7 +104,6 @@ empty the batch so that you can add more rows to it if you want. Note
104
104
  that all records saved together will have the same created_at/updated_at
105
105
  timestamp (unless one was explicitly set).
106
106
 
107
-
108
107
  ### Batch Set Size
109
108
 
110
109
  By default, the size of the insert is limited to 500 rows at a time.
@@ -149,22 +148,29 @@ Book.bulk_insert(*destination_columns, ignore: true) do |worker|
149
148
  end
150
149
  ```
151
150
 
152
- ### Update Duplicates (MySQL)
151
+ ### Update Duplicates (MySQL, PostgreSQL)
153
152
 
154
153
  If you don't want to ignore duplicate rows but instead want to update them
155
154
  then you can use the _update_duplicates_ option. Set this option to true
156
- and when a duplicate row is found the row will be updated with your new
157
- values. Default value for this option is false.
155
+ (MySQL) or list unique column names (PostgreSQL) and when a duplicate row
156
+ is found the row will be updated with your new values.
157
+ Default value for this option is false.
158
158
 
159
159
  ```ruby
160
160
  destination_columns = [:title, :author]
161
161
 
162
- # Update duplicate rows
162
+ # Update duplicate rows (MySQL)
163
163
  Book.bulk_insert(*destination_columns, update_duplicates: true) do |worker|
164
164
  worker.add(...)
165
165
  worker.add(...)
166
166
  # ...
167
167
  end
168
+
169
+ # Update duplicate rows (PostgreSQL)
170
+ Book.bulk_insert(*destination_columns, update_duplicates: %w[title]) do |worker|
171
+ worker.add(...)
172
+ # ...
173
+ end
168
174
  ```
169
175
 
170
176
  ### Return Primary Keys (PostgreSQL, PostGIS)
@@ -185,7 +191,6 @@ end
185
191
  worker.result_sets
186
192
  ```
187
193
 
188
-
189
194
  ## License
190
195
 
191
196
  BulkInsert is released under the MIT license (see MIT-LICENSE) by
@@ -0,0 +1,22 @@
1
+ require_relative 'statement_adapters/generic_adapter'
2
+ require_relative 'statement_adapters/mysql_adapter'
3
+ require_relative 'statement_adapters/postgresql_adapter'
4
+ require_relative 'statement_adapters/sqlite_adapter'
5
+
6
+ module BulkInsert
7
+ module StatementAdapters
8
+ def adapter_for(connection)
9
+ case connection.adapter_name
10
+ when /^mysql/i
11
+ MySQLAdapter.new
12
+ when /\APost(?:greSQL|GIS)/i
13
+ PostgreSQLAdapter.new
14
+ when /\ASQLite/i
15
+ SQLiteAdapter.new
16
+ else
17
+ GenericAdapter.new
18
+ end
19
+ end
20
+ module_function :adapter_for
21
+ end
22
+ end
@@ -0,0 +1,21 @@
1
+ module BulkInsert
2
+ module StatementAdapters
3
+ class BaseAdapter
4
+ def initialize
5
+ raise "You cannot initialize base adapter" if self.class == BaseAdapter
6
+ end
7
+
8
+ def insert_ignore_statement
9
+ raise "Not implemented"
10
+ end
11
+
12
+ def on_conflict_statement(_columns, _ignore, _update_duplicates)
13
+ raise "Not implemented"
14
+ end
15
+
16
+ def primary_key_return_statement(_primary_key)
17
+ raise "Not implemented"
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,19 @@
1
+ require_relative 'base_adapter'
2
+
3
+ module BulkInsert
4
+ module StatementAdapters
5
+ class GenericAdapter < BaseAdapter
6
+ def insert_ignore_statement
7
+ ''
8
+ end
9
+
10
+ def on_conflict_statement(_columns, _ignore, _update_duplicates)
11
+ ''
12
+ end
13
+
14
+ def primary_key_return_statement(_primary_key)
15
+ ''
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,24 @@
1
+ require_relative 'base_adapter'
2
+
3
+ module BulkInsert
4
+ module StatementAdapters
5
+ class MySQLAdapter < BaseAdapter
6
+ def insert_ignore_statement
7
+ 'IGNORE'
8
+ end
9
+
10
+ def on_conflict_statement(columns, _ignore, update_duplicates)
11
+ return '' unless update_duplicates
12
+
13
+ update_values = columns.map do |column|
14
+ "`#{column.name}`=VALUES(`#{column.name}`)"
15
+ end.join(', ')
16
+ ' ON DUPLICATE KEY UPDATE ' + update_values
17
+ end
18
+
19
+ def primary_key_return_statement(_primary_key)
20
+ ''
21
+ end
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,26 @@
1
+ require_relative 'base_adapter'
2
+
3
+ module BulkInsert
4
+ module StatementAdapters
5
+ class PostgreSQLAdapter < BaseAdapter
6
+ def insert_ignore_statement
7
+ ''
8
+ end
9
+
10
+ def on_conflict_statement(columns, ignore, update_duplicates)
11
+ if ignore
12
+ ' ON CONFLICT DO NOTHING'
13
+ elsif update_duplicates
14
+ update_values = columns.map do |column|
15
+ "#{column.name}=EXCLUDED.#{column.name}"
16
+ end.join(', ')
17
+ ' ON CONFLICT(' + update_duplicates.join(', ') + ') DO UPDATE SET ' + update_values
18
+ end
19
+ end
20
+
21
+ def primary_key_return_statement(primary_key)
22
+ " RETURNING #{primary_key}"
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,19 @@
1
+ require_relative 'base_adapter'
2
+
3
+ module BulkInsert
4
+ module StatementAdapters
5
+ class SQLiteAdapter < BaseAdapter
6
+ def insert_ignore_statement
7
+ 'OR IGNORE'
8
+ end
9
+
10
+ def on_conflict_statement(_columns, _ignore, _update_duplicates)
11
+ ''
12
+ end
13
+
14
+ def primary_key_return_statement(_primary_key)
15
+ ''
16
+ end
17
+ end
18
+ end
19
+ end
@@ -1,6 +1,6 @@
1
1
  module BulkInsert
2
2
  MAJOR = 1
3
- MINOR = 7
3
+ MINOR = 8
4
4
  TINY = 0
5
5
 
6
6
  VERSION = [MAJOR, MINOR, TINY].join(".")
@@ -1,3 +1,5 @@
1
+ require_relative 'statement_adapters'
2
+
1
3
  module BulkInsert
2
4
  class Worker
3
5
  attr_reader :connection
@@ -8,6 +10,8 @@ module BulkInsert
8
10
  attr_reader :ignore, :update_duplicates, :result_sets
9
11
 
10
12
  def initialize(connection, table_name, primary_key, column_names, set_size=500, ignore=false, update_duplicates=false, return_primary_keys=false)
13
+ @statement_adapter = StatementAdapters.adapter_for(connection)
14
+
11
15
  @connection = connection
12
16
  @set_size = set_size
13
17
 
@@ -116,8 +120,8 @@ module BulkInsert
116
120
 
117
121
  if !rows.empty?
118
122
  sql << rows.join(",")
119
- sql << on_conflict_statement
120
- sql << primary_key_return_statement
123
+ sql << @statement_adapter.on_conflict_statement(@columns, ignore, update_duplicates)
124
+ sql << @statement_adapter.primary_key_return_statement(@primary_key) if @return_primary_keys
121
125
  sql
122
126
  else
123
127
  false
@@ -125,41 +129,8 @@ module BulkInsert
125
129
  end
126
130
 
127
131
  def insert_sql_statement
132
+ insert_ignore = @ignore ? @statement_adapter.insert_ignore_statement : ''
128
133
  "INSERT #{insert_ignore} INTO #{@table_name} (#{@column_names}) VALUES "
129
134
  end
130
-
131
- def insert_ignore
132
- if ignore
133
- case adapter_name
134
- when /^mysql/i
135
- 'IGNORE'
136
- when /\ASQLite/i # SQLite
137
- 'OR IGNORE'
138
- else
139
- '' # Not supported
140
- end
141
- end
142
- end
143
-
144
- def primary_key_return_statement
145
- if @return_primary_keys && adapter_name =~ /\APost(?:greSQL|GIS)/i
146
- " RETURNING #{@primary_key}"
147
- else
148
- ''
149
- end
150
- end
151
-
152
- def on_conflict_statement
153
- if (adapter_name =~ /\APost(?:greSQL|GIS)/i && ignore )
154
- ' ON CONFLICT DO NOTHING'
155
- elsif adapter_name =~ /^mysql/i && update_duplicates
156
- update_values = @columns.map do |column|
157
- "`#{column.name}`=VALUES(`#{column.name}`)"
158
- end.join(', ')
159
- ' ON DUPLICATE KEY UPDATE ' + update_values
160
- else
161
- ''
162
- end
163
- end
164
135
  end
165
136
  end
@@ -1,3 +1,4 @@
1
+ require 'minitest/mock'
1
2
  require 'test_helper'
2
3
 
3
4
  class BulkInsertWorkerTest < ActiveSupport::TestCase
@@ -270,93 +271,124 @@ class BulkInsertWorkerTest < ActiveSupport::TestCase
270
271
  end
271
272
 
272
273
  test "adapter dependent mysql methods" do
273
- mysql_worker = BulkInsert::Worker.new(
274
- Testing.connection,
275
- Testing.table_name,
276
- 'id',
277
- %w(greeting age happy created_at updated_at color),
278
- 500, # batch size
279
- true) # ignore
280
- mysql_worker.adapter_name = 'MySQL'
281
-
282
- assert_equal mysql_worker.adapter_name, 'MySQL'
283
- assert_equal (mysql_worker.adapter_name == 'MySQL'), true
284
- assert_equal mysql_worker.ignore, true
285
- assert_equal ((mysql_worker.adapter_name == 'MySQL') & mysql_worker.ignore), true
286
-
287
- mysql_worker.add ["Yo", 15, false, nil, nil]
288
-
289
- assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')"
274
+ connection = Testing.connection
275
+ connection.stub :adapter_name, 'MySQL' do
276
+ mysql_worker = BulkInsert::Worker.new(
277
+ connection,
278
+ Testing.table_name,
279
+ 'id',
280
+ %w(greeting age happy created_at updated_at color),
281
+ 500, # batch size
282
+ true # ignore
283
+ )
284
+
285
+ assert_equal mysql_worker.adapter_name, 'MySQL'
286
+ assert_equal (mysql_worker.adapter_name == 'MySQL'), true
287
+ assert_equal mysql_worker.ignore, true
288
+ assert_equal ((mysql_worker.adapter_name == 'MySQL') & mysql_worker.ignore), true
289
+
290
+ mysql_worker.add ["Yo", 15, false, nil, nil]
291
+
292
+ assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')"
293
+ end
290
294
  end
291
295
 
292
296
  test "adapter dependent mysql methods work for mysql2" do
293
- mysql_worker = BulkInsert::Worker.new(
294
- Testing.connection,
295
- Testing.table_name,
296
- 'id',
297
- %w(greeting age happy created_at updated_at color),
298
- 500, # batch size
299
- true, # ignore
300
- true) # update_duplicates
301
- mysql_worker.adapter_name = 'Mysql2'
302
-
303
- assert_equal mysql_worker.adapter_name, 'Mysql2'
304
- assert mysql_worker.ignore
305
-
306
- mysql_worker.add ["Yo", 15, false, nil, nil]
307
-
308
- assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)"
297
+ connection = Testing.connection
298
+ connection.stub :adapter_name, 'Mysql2' do
299
+ mysql_worker = BulkInsert::Worker.new(
300
+ connection,
301
+ Testing.table_name,
302
+ 'id',
303
+ %w(greeting age happy created_at updated_at color),
304
+ 500, # batch size
305
+ true, # ignore
306
+ true) # update_duplicates
307
+
308
+ assert_equal mysql_worker.adapter_name, 'Mysql2'
309
+ assert mysql_worker.ignore
310
+
311
+ mysql_worker.add ["Yo", 15, false, nil, nil]
312
+
313
+ assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)"
314
+ end
309
315
  end
310
316
 
311
317
  test "adapter dependent Mysql2Spatial methods" do
312
- mysql_worker = BulkInsert::Worker.new(
313
- Testing.connection,
314
- Testing.table_name,
315
- 'id',
316
- %w(greeting age happy created_at updated_at color),
317
- 500, # batch size
318
- true) # ignore
319
- mysql_worker.adapter_name = 'Mysql2Spatial'
318
+ connection = Testing.connection
319
+ connection.stub :adapter_name, 'Mysql2Spatial' do
320
+ mysql_worker = BulkInsert::Worker.new(
321
+ connection,
322
+ Testing.table_name,
323
+ 'id',
324
+ %w(greeting age happy created_at updated_at color),
325
+ 500, # batch size
326
+ true) # ignore
320
327
 
321
- assert_equal mysql_worker.adapter_name, 'Mysql2Spatial'
328
+ assert_equal mysql_worker.adapter_name, 'Mysql2Spatial'
322
329
 
323
- mysql_worker.add ["Yo", 15, false, nil, nil]
330
+ mysql_worker.add ["Yo", 15, false, nil, nil]
324
331
 
325
- assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')"
332
+ assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')"
333
+ end
326
334
  end
327
335
 
328
336
  test "adapter dependent postgresql methods" do
329
- pgsql_worker = BulkInsert::Worker.new(
330
- Testing.connection,
331
- Testing.table_name,
332
- 'id',
333
- %w(greeting age happy created_at updated_at color),
334
- 500, # batch size
335
- true, # ignore
336
- false, # update duplicates
337
- true # return primary keys
338
- )
339
- pgsql_worker.adapter_name = 'PostgreSQL'
340
- pgsql_worker.add ["Yo", 15, false, nil, nil]
337
+ connection = Testing.connection
338
+ connection.stub :adapter_name, 'PostgreSQL' do
339
+ pgsql_worker = BulkInsert::Worker.new(
340
+ connection,
341
+ Testing.table_name,
342
+ 'id',
343
+ %w(greeting age happy created_at updated_at color),
344
+ 500, # batch size
345
+ true, # ignore
346
+ false, # update duplicates
347
+ true # return primary keys
348
+ )
349
+
350
+ pgsql_worker.add ["Yo", 15, false, nil, nil]
351
+
352
+ assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id"
353
+ end
354
+ end
341
355
 
342
- assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id"
356
+ test "adapter dependent postgresql methods (with update_duplicates)" do
357
+ connection = Testing.connection
358
+ connection.stub :adapter_name, 'PostgreSQL' do
359
+ pgsql_worker = BulkInsert::Worker.new(
360
+ connection,
361
+ Testing.table_name,
362
+ 'id',
363
+ %w(greeting age happy created_at updated_at color),
364
+ 500, # batch size
365
+ false, # ignore
366
+ %w(greeting age happy), # update duplicates
367
+ true # return primary keys
368
+ )
369
+ pgsql_worker.add ["Yo", 15, false, nil, nil]
370
+
371
+ assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON CONFLICT(greeting, age, happy) DO UPDATE SET greeting=EXCLUDED.greeting, age=EXCLUDED.age, happy=EXCLUDED.happy, created_at=EXCLUDED.created_at, updated_at=EXCLUDED.updated_at, color=EXCLUDED.color RETURNING id"
372
+ end
343
373
  end
344
374
 
345
375
  test "adapter dependent PostGIS methods" do
346
- pgsql_worker = BulkInsert::Worker.new(
347
- Testing.connection,
348
- Testing.table_name,
349
- 'id',
350
- %w(greeting age happy created_at updated_at color),
351
- 500, # batch size
352
- true, # ignore
353
- false, # update duplicates
354
- true # return primary keys
355
- )
356
- pgsql_worker.adapter_name = 'PostGIS'
357
- pgsql_worker.add ["Yo", 15, false, nil, nil]
358
-
359
- assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id"
376
+ connection = Testing.connection
377
+ connection.stub :adapter_name, 'PostGIS' do
378
+ pgsql_worker = BulkInsert::Worker.new(
379
+ connection,
380
+ Testing.table_name,
381
+ 'id',
382
+ %w(greeting age happy created_at updated_at color),
383
+ 500, # batch size
384
+ true, # ignore
385
+ false, # update duplicates
386
+ true # return primary keys
387
+ )
388
+ pgsql_worker.add ["Yo", 15, false, nil, nil]
389
+
390
+ assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id"
391
+ end
360
392
  end
361
393
 
362
394
  test "adapter dependent sqlite3 methods (with lowercase adapter name)" do
@@ -388,17 +420,20 @@ class BulkInsertWorkerTest < ActiveSupport::TestCase
388
420
  end
389
421
 
390
422
  test "mysql adapter can update duplicates" do
391
- mysql_worker = BulkInsert::Worker.new(
392
- Testing.connection,
393
- Testing.table_name,
394
- 'id',
395
- %w(greeting age happy created_at updated_at color),
396
- 500, # batch size
397
- false, # ignore
398
- true) # update_duplicates
399
- mysql_worker.adapter_name = 'MySQL'
400
- mysql_worker.add ["Yo", 15, false, nil, nil]
401
-
402
- assert_equal mysql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)"
423
+ connection = Testing.connection
424
+ connection.stub :adapter_name, 'MySQL' do
425
+ mysql_worker = BulkInsert::Worker.new(
426
+ connection,
427
+ Testing.table_name,
428
+ 'id',
429
+ %w(greeting age happy created_at updated_at color),
430
+ 500, # batch size
431
+ false, # ignore
432
+ true # update_duplicates
433
+ )
434
+ mysql_worker.add ["Yo", 15, false, nil, nil]
435
+
436
+ assert_equal mysql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)"
437
+ end
403
438
  end
404
439
  end