bulk_insert 1.6.0 → 1.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/README.md +52 -5
- data/lib/bulk_insert.rb +2 -2
- data/lib/bulk_insert/statement_adapters.rb +22 -0
- data/lib/bulk_insert/statement_adapters/base_adapter.rb +21 -0
- data/lib/bulk_insert/statement_adapters/generic_adapter.rb +19 -0
- data/lib/bulk_insert/statement_adapters/mysql_adapter.rb +24 -0
- data/lib/bulk_insert/statement_adapters/postgresql_adapter.rb +28 -0
- data/lib/bulk_insert/statement_adapters/sqlite_adapter.rb +19 -0
- data/lib/bulk_insert/version.rb +1 -1
- data/lib/bulk_insert/worker.rb +35 -31
- data/test/bulk_insert/worker_test.rb +280 -105
- data/test/bulk_insert_test.rb +23 -1
- data/test/connection_mocks.rb +140 -0
- data/test/dummy/app/assets/config/manifest.js +0 -0
- data/test/dummy/config/application.rb +44 -3
- data/test/dummy/config/database.yml +15 -19
- data/test/dummy/config/environments/test.rb +1 -1
- data/test/dummy/db/test.sqlite3 +0 -0
- data/test/dummy/log/test.log +20733 -4605
- metadata +48 -55
- data/test/dummy/db/development.sqlite3 +0 -0
- data/test/dummy/log/development.log +0 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 385a28475e8a07cdf1cad85aacf241734ce65ed922648417c7beda2b88a5b3c9
|
4
|
+
data.tar.gz: a14a05da23f256f3d30cf69fab032ec72c5953fe1ddb3ccc45a0673bef717266
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cbe45a91a166d6faa3e2899a4fad7ac1af8a6e050249adbc810121ddc3000f0786c277b8c240809935bbb80178bbb1df43044dda32a02cea87c0d5d956e3cee1
|
7
|
+
data.tar.gz: 165de20f26c00dfcbe605866576f731199f692b7bccb07e671b8dcb9541552e39d0940a9f6fff73701286586f909e7be1254be6df157f41f76e83b105e88b9f2
|
data/README.md
CHANGED
@@ -104,7 +104,6 @@ empty the batch so that you can add more rows to it if you want. Note
|
|
104
104
|
that all records saved together will have the same created_at/updated_at
|
105
105
|
timestamp (unless one was explicitly set).
|
106
106
|
|
107
|
-
|
108
107
|
### Batch Set Size
|
109
108
|
|
110
109
|
By default, the size of the insert is limited to 500 rows at a time.
|
@@ -149,24 +148,72 @@ Book.bulk_insert(*destination_columns, ignore: true) do |worker|
|
|
149
148
|
end
|
150
149
|
```
|
151
150
|
|
152
|
-
### Update Duplicates (MySQL)
|
151
|
+
### Update Duplicates (MySQL, PostgreSQL)
|
153
152
|
|
154
153
|
If you don't want to ignore duplicate rows but instead want to update them
|
155
154
|
then you can use the _update_duplicates_ option. Set this option to true
|
156
|
-
|
157
|
-
|
155
|
+
(MySQL) or list unique column names (PostgreSQL) and when a duplicate row
|
156
|
+
is found the row will be updated with your new values.
|
157
|
+
Default value for this option is false.
|
158
158
|
|
159
159
|
```ruby
|
160
160
|
destination_columns = [:title, :author]
|
161
161
|
|
162
|
-
# Update duplicate rows
|
162
|
+
# Update duplicate rows (MySQL)
|
163
163
|
Book.bulk_insert(*destination_columns, update_duplicates: true) do |worker|
|
164
164
|
worker.add(...)
|
165
165
|
worker.add(...)
|
166
166
|
# ...
|
167
167
|
end
|
168
|
+
|
169
|
+
# Update duplicate rows (PostgreSQL)
|
170
|
+
Book.bulk_insert(*destination_columns, update_duplicates: %w[title]) do |worker|
|
171
|
+
worker.add(...)
|
172
|
+
# ...
|
173
|
+
end
|
174
|
+
```
|
175
|
+
|
176
|
+
### Return Primary Keys (PostgreSQL, PostGIS)
|
177
|
+
|
178
|
+
If you want the worker to store primary keys of inserted records, then you can
|
179
|
+
use the _return_primary_keys_ option. The worker will store a `result_sets`
|
180
|
+
array of `ActiveRecord::Result` objects. Each `ActiveRecord::Result` object
|
181
|
+
will contain the primary keys of a batch of inserted records.
|
182
|
+
|
183
|
+
```ruby
|
184
|
+
worker = Book.bulk_insert(*destination_columns, return_primary_keys: true) do
|
185
|
+
|worker|
|
186
|
+
worker.add(...)
|
187
|
+
worker.add(...)
|
188
|
+
# ...
|
189
|
+
end
|
190
|
+
|
191
|
+
worker.result_sets
|
168
192
|
```
|
169
193
|
|
194
|
+
## Ruby and Rails Versions Supported
|
195
|
+
|
196
|
+
> :warning: The scope of this gem may be somehow covered natively by the `.insert_all` API
|
197
|
+
> introduced by [Rails 6](https://apidock.com/rails/v6.0.0/ActiveRecord/Persistence/ClassMethods/insert_all).
|
198
|
+
> This gem represents the state of art for rails version < 6 and it is still open to
|
199
|
+
> further developments for more recent versions.
|
200
|
+
|
201
|
+
The current CI prevents regressions on the following versions:
|
202
|
+
|
203
|
+
ruby / rails | `~>3` | `~>4` | `~>5` | `~>6`
|
204
|
+
:-----------:|-------|-------|-------|------
|
205
|
+
2.2 | yes | yes | no | no
|
206
|
+
2.3 | yes | yes | yes | no
|
207
|
+
2.4 | no | yes | yes | no
|
208
|
+
2.5 | no | no | yes | yes
|
209
|
+
2.6 | no | no | yes | yes
|
210
|
+
2.7 | no | no | yes | yes
|
211
|
+
|
212
|
+
The adapters covered in the CI are:
|
213
|
+
* sqlite
|
214
|
+
* mysql
|
215
|
+
* postgresql
|
216
|
+
|
170
217
|
|
171
218
|
## License
|
172
219
|
|
data/lib/bulk_insert.rb
CHANGED
@@ -4,9 +4,9 @@ module BulkInsert
|
|
4
4
|
extend ActiveSupport::Concern
|
5
5
|
|
6
6
|
module ClassMethods
|
7
|
-
def bulk_insert(*columns, values: nil, set_size:500, ignore: false, update_duplicates: false)
|
7
|
+
def bulk_insert(*columns, values: nil, set_size:500, ignore: false, update_duplicates: false, return_primary_keys: false)
|
8
8
|
columns = default_bulk_columns if columns.empty?
|
9
|
-
worker = BulkInsert::Worker.new(connection, table_name, columns, set_size, ignore, update_duplicates)
|
9
|
+
worker = BulkInsert::Worker.new(connection, table_name, primary_key, columns, set_size, ignore, update_duplicates, return_primary_keys)
|
10
10
|
|
11
11
|
if values.present?
|
12
12
|
transaction do
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require_relative 'statement_adapters/generic_adapter'
|
2
|
+
require_relative 'statement_adapters/mysql_adapter'
|
3
|
+
require_relative 'statement_adapters/postgresql_adapter'
|
4
|
+
require_relative 'statement_adapters/sqlite_adapter'
|
5
|
+
|
6
|
+
module BulkInsert
|
7
|
+
module StatementAdapters
|
8
|
+
def adapter_for(connection)
|
9
|
+
case connection.adapter_name
|
10
|
+
when /^mysql/i
|
11
|
+
MySQLAdapter.new
|
12
|
+
when /\APost(?:greSQL|GIS)/i
|
13
|
+
PostgreSQLAdapter.new
|
14
|
+
when /\ASQLite/i
|
15
|
+
SQLiteAdapter.new
|
16
|
+
else
|
17
|
+
GenericAdapter.new
|
18
|
+
end
|
19
|
+
end
|
20
|
+
module_function :adapter_for
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module BulkInsert
|
2
|
+
module StatementAdapters
|
3
|
+
class BaseAdapter
|
4
|
+
def initialize
|
5
|
+
raise "You cannot initialize base adapter" if self.class == BaseAdapter
|
6
|
+
end
|
7
|
+
|
8
|
+
def insert_ignore_statement
|
9
|
+
raise "Not implemented"
|
10
|
+
end
|
11
|
+
|
12
|
+
def on_conflict_statement(_columns, _ignore, _update_duplicates)
|
13
|
+
raise "Not implemented"
|
14
|
+
end
|
15
|
+
|
16
|
+
def primary_key_return_statement(_primary_key)
|
17
|
+
raise "Not implemented"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require_relative 'base_adapter'
|
2
|
+
|
3
|
+
module BulkInsert
|
4
|
+
module StatementAdapters
|
5
|
+
class GenericAdapter < BaseAdapter
|
6
|
+
def insert_ignore_statement
|
7
|
+
''
|
8
|
+
end
|
9
|
+
|
10
|
+
def on_conflict_statement(_columns, _ignore, _update_duplicates)
|
11
|
+
''
|
12
|
+
end
|
13
|
+
|
14
|
+
def primary_key_return_statement(_primary_key)
|
15
|
+
''
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require_relative 'base_adapter'
|
2
|
+
|
3
|
+
module BulkInsert
|
4
|
+
module StatementAdapters
|
5
|
+
class MySQLAdapter < BaseAdapter
|
6
|
+
def insert_ignore_statement
|
7
|
+
'IGNORE'
|
8
|
+
end
|
9
|
+
|
10
|
+
def on_conflict_statement(columns, _ignore, update_duplicates)
|
11
|
+
return '' unless update_duplicates
|
12
|
+
|
13
|
+
update_values = columns.map do |column|
|
14
|
+
"`#{column.name}`=VALUES(`#{column.name}`)"
|
15
|
+
end.join(', ')
|
16
|
+
' ON DUPLICATE KEY UPDATE ' + update_values
|
17
|
+
end
|
18
|
+
|
19
|
+
def primary_key_return_statement(_primary_key)
|
20
|
+
''
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require_relative 'base_adapter'
|
2
|
+
|
3
|
+
module BulkInsert
|
4
|
+
module StatementAdapters
|
5
|
+
class PostgreSQLAdapter < BaseAdapter
|
6
|
+
def insert_ignore_statement
|
7
|
+
''
|
8
|
+
end
|
9
|
+
|
10
|
+
def on_conflict_statement(columns, ignore, update_duplicates)
|
11
|
+
if ignore
|
12
|
+
' ON CONFLICT DO NOTHING'
|
13
|
+
elsif update_duplicates
|
14
|
+
update_values = columns.map do |column|
|
15
|
+
"#{column.name}=EXCLUDED.#{column.name}"
|
16
|
+
end.join(', ')
|
17
|
+
' ON CONFLICT(' + update_duplicates.join(', ') + ') DO UPDATE SET ' + update_values
|
18
|
+
else
|
19
|
+
''
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def primary_key_return_statement(primary_key)
|
24
|
+
" RETURNING #{primary_key}"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require_relative 'base_adapter'
|
2
|
+
|
3
|
+
module BulkInsert
|
4
|
+
module StatementAdapters
|
5
|
+
class SQLiteAdapter < BaseAdapter
|
6
|
+
def insert_ignore_statement
|
7
|
+
'OR IGNORE'
|
8
|
+
end
|
9
|
+
|
10
|
+
def on_conflict_statement(_columns, _ignore, _update_duplicates)
|
11
|
+
''
|
12
|
+
end
|
13
|
+
|
14
|
+
def primary_key_return_statement(_primary_key)
|
15
|
+
''
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
data/lib/bulk_insert/version.rb
CHANGED
data/lib/bulk_insert/worker.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
require_relative 'statement_adapters'
|
2
|
+
|
1
3
|
module BulkInsert
|
2
4
|
class Worker
|
3
5
|
attr_reader :connection
|
@@ -5,9 +7,11 @@ module BulkInsert
|
|
5
7
|
attr_accessor :before_save_callback
|
6
8
|
attr_accessor :after_save_callback
|
7
9
|
attr_accessor :adapter_name
|
8
|
-
attr_reader :ignore, :update_duplicates
|
10
|
+
attr_reader :ignore, :update_duplicates, :result_sets
|
11
|
+
|
12
|
+
def initialize(connection, table_name, primary_key, column_names, set_size=500, ignore=false, update_duplicates=false, return_primary_keys=false)
|
13
|
+
@statement_adapter = StatementAdapters.adapter_for(connection)
|
9
14
|
|
10
|
-
def initialize(connection, table_name, column_names, set_size=500, ignore=false, update_duplicates=false)
|
11
15
|
@connection = connection
|
12
16
|
@set_size = set_size
|
13
17
|
|
@@ -15,10 +19,12 @@ module BulkInsert
|
|
15
19
|
# INSERT IGNORE only fails inserts with duplicate keys or unallowed nulls not the whole set of inserts
|
16
20
|
@ignore = ignore
|
17
21
|
@update_duplicates = update_duplicates
|
22
|
+
@return_primary_keys = return_primary_keys
|
18
23
|
|
19
24
|
columns = connection.columns(table_name)
|
20
25
|
column_map = columns.inject({}) { |h, c| h.update(c.name => c) }
|
21
26
|
|
27
|
+
@primary_key = primary_key
|
22
28
|
@columns = column_names.map { |name| column_map[name.to_s] }
|
23
29
|
@table_name = connection.quote_table_name(table_name)
|
24
30
|
@column_names = column_names.map { |name| connection.quote_column_name(name) }.join(",")
|
@@ -26,6 +32,7 @@ module BulkInsert
|
|
26
32
|
@before_save_callback = nil
|
27
33
|
@after_save_callback = nil
|
28
34
|
|
35
|
+
@result_sets = []
|
29
36
|
@set = []
|
30
37
|
end
|
31
38
|
|
@@ -76,7 +83,7 @@ module BulkInsert
|
|
76
83
|
def save!
|
77
84
|
if pending?
|
78
85
|
@before_save_callback.(@set) if @before_save_callback
|
79
|
-
|
86
|
+
execute_query
|
80
87
|
@after_save_callback.() if @after_save_callback
|
81
88
|
@set.clear
|
82
89
|
end
|
@@ -84,6 +91,24 @@ module BulkInsert
|
|
84
91
|
self
|
85
92
|
end
|
86
93
|
|
94
|
+
def execute_query
|
95
|
+
if query = compose_insert_query
|
96
|
+
|
97
|
+
# Return primary key support broke mysql compatibility
|
98
|
+
# with rails < 5 mysql adapter. (see issue #41)
|
99
|
+
if ActiveRecord::VERSION::STRING < "5.0.0" && @statement_adapter.is_a?(StatementAdapters::MySQLAdapter)
|
100
|
+
# raise an exception for unsupported return_primary_keys
|
101
|
+
raise ArgumentError.new("BulkInsert does not support @return_primary_keys for mysql and rails < 5") if @return_primary_keys
|
102
|
+
|
103
|
+
# restore v1.6 query execution
|
104
|
+
@connection.execute(query)
|
105
|
+
else
|
106
|
+
result_set = @connection.exec_query(query)
|
107
|
+
@result_sets.push(result_set) if @return_primary_keys
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
87
112
|
def compose_insert_query
|
88
113
|
sql = insert_sql_statement
|
89
114
|
@now = Time.now
|
@@ -95,7 +120,10 @@ module BulkInsert
|
|
95
120
|
value = @now if value == :__timestamp_placeholder
|
96
121
|
|
97
122
|
if ActiveRecord::VERSION::STRING >= "5.0.0"
|
98
|
-
|
123
|
+
if column
|
124
|
+
type = @connection.lookup_cast_type_from_column(column)
|
125
|
+
value = type.serialize(value)
|
126
|
+
end
|
99
127
|
values << @connection.quote(value)
|
100
128
|
else
|
101
129
|
values << @connection.quote(value, column)
|
@@ -106,7 +134,8 @@ module BulkInsert
|
|
106
134
|
|
107
135
|
if !rows.empty?
|
108
136
|
sql << rows.join(",")
|
109
|
-
sql << on_conflict_statement
|
137
|
+
sql << @statement_adapter.on_conflict_statement(@columns, ignore, update_duplicates)
|
138
|
+
sql << @statement_adapter.primary_key_return_statement(@primary_key) if @return_primary_keys
|
110
139
|
sql
|
111
140
|
else
|
112
141
|
false
|
@@ -114,33 +143,8 @@ module BulkInsert
|
|
114
143
|
end
|
115
144
|
|
116
145
|
def insert_sql_statement
|
146
|
+
insert_ignore = @ignore ? @statement_adapter.insert_ignore_statement : ''
|
117
147
|
"INSERT #{insert_ignore} INTO #{@table_name} (#{@column_names}) VALUES "
|
118
148
|
end
|
119
|
-
|
120
|
-
def insert_ignore
|
121
|
-
if ignore
|
122
|
-
case adapter_name
|
123
|
-
when /^mysql/i
|
124
|
-
'IGNORE'
|
125
|
-
when /\ASQLite/i # SQLite
|
126
|
-
'OR IGNORE'
|
127
|
-
else
|
128
|
-
'' # Not supported
|
129
|
-
end
|
130
|
-
end
|
131
|
-
end
|
132
|
-
|
133
|
-
def on_conflict_statement
|
134
|
-
if (adapter_name =~ /\APost(?:greSQL|GIS)/i && ignore )
|
135
|
-
' ON CONFLICT DO NOTHING'
|
136
|
-
elsif adapter_name =~ /^mysql/i && update_duplicates
|
137
|
-
update_values = @columns.map do |column|
|
138
|
-
"#{column.name}=VALUES(#{column.name})"
|
139
|
-
end.join(', ')
|
140
|
-
' ON DUPLICATE KEY UPDATE ' + update_values
|
141
|
-
else
|
142
|
-
''
|
143
|
-
end
|
144
|
-
end
|
145
149
|
end
|
146
150
|
end
|
@@ -1,12 +1,17 @@
|
|
1
|
+
require 'minitest/mock'
|
1
2
|
require 'test_helper'
|
3
|
+
require 'connection_mocks'
|
2
4
|
|
3
5
|
class BulkInsertWorkerTest < ActiveSupport::TestCase
|
6
|
+
include ConnectionMocks
|
7
|
+
|
4
8
|
setup do
|
5
9
|
@insert = BulkInsert::Worker.new(
|
6
10
|
Testing.connection,
|
7
11
|
Testing.table_name,
|
12
|
+
'id',
|
8
13
|
%w(greeting age happy created_at updated_at color))
|
9
|
-
@now = Time.now
|
14
|
+
@now = Time.now.utc
|
10
15
|
end
|
11
16
|
|
12
17
|
test "empty insert is not pending" do
|
@@ -35,8 +40,8 @@ class BulkInsertWorkerTest < ActiveSupport::TestCase
|
|
35
40
|
@insert.save!
|
36
41
|
|
37
42
|
record = Testing.first
|
38
|
-
assert_operator record.created_at, :>=, now
|
39
|
-
assert_operator record.updated_at, :>=, now
|
43
|
+
assert_operator record.created_at.to_i, :>=, now.to_i
|
44
|
+
assert_operator record.updated_at.to_i, :>=, now.to_i
|
40
45
|
end
|
41
46
|
|
42
47
|
test "default timestamp columns should be equivalent for the entire batch" do
|
@@ -109,8 +114,8 @@ class BulkInsertWorkerTest < ActiveSupport::TestCase
|
|
109
114
|
@insert.add ["Hello", 25, true, @now, @now]
|
110
115
|
@insert.save!
|
111
116
|
|
112
|
-
yo = Testing.
|
113
|
-
hello = Testing.
|
117
|
+
yo = Testing.where(greeting: 'Yo').first
|
118
|
+
hello = Testing.where(greeting: 'Hello').first
|
114
119
|
|
115
120
|
assert_not_nil yo
|
116
121
|
assert_equal 15, yo.age
|
@@ -121,6 +126,57 @@ class BulkInsertWorkerTest < ActiveSupport::TestCase
|
|
121
126
|
assert_equal true, hello.happy?
|
122
127
|
end
|
123
128
|
|
129
|
+
test "save! does not add to result sets when not returning primary keys" do
|
130
|
+
@insert.add greeting: "first"
|
131
|
+
@insert.add greeting: "second"
|
132
|
+
@insert.save!
|
133
|
+
|
134
|
+
assert_equal 0, @insert.result_sets.count
|
135
|
+
end
|
136
|
+
|
137
|
+
|
138
|
+
test "save! adds to result sets when returning primary keys" do
|
139
|
+
worker = BulkInsert::Worker.new(
|
140
|
+
Testing.connection,
|
141
|
+
Testing.table_name,
|
142
|
+
'id',
|
143
|
+
%w(greeting age happy created_at updated_at color),
|
144
|
+
500,
|
145
|
+
false,
|
146
|
+
false,
|
147
|
+
true
|
148
|
+
)
|
149
|
+
|
150
|
+
# return_primary_keys is not supported for mysql and rails < 5
|
151
|
+
# skip is not supported in the minitest version used for testing rails 3
|
152
|
+
return if ActiveRecord::VERSION::STRING < "5.0.0" && worker.adapter_name =~ /^mysql/i
|
153
|
+
|
154
|
+
assert_no_difference -> { worker.result_sets.count } do
|
155
|
+
worker.save!
|
156
|
+
end
|
157
|
+
|
158
|
+
worker.add greeting: "first"
|
159
|
+
worker.add greeting: "second"
|
160
|
+
worker.save!
|
161
|
+
assert_equal 1, worker.result_sets.count
|
162
|
+
|
163
|
+
worker.add greeting: "third"
|
164
|
+
worker.add greeting: "fourth"
|
165
|
+
worker.save!
|
166
|
+
assert_equal 2, worker.result_sets.count
|
167
|
+
end
|
168
|
+
|
169
|
+
test "initialized with empty result sets array" do
|
170
|
+
new_worker = BulkInsert::Worker.new(
|
171
|
+
Testing.connection,
|
172
|
+
Testing.table_name,
|
173
|
+
'id',
|
174
|
+
%w(greeting age happy created_at updated_at color)
|
175
|
+
)
|
176
|
+
assert_instance_of(Array, new_worker.result_sets)
|
177
|
+
assert_empty new_worker.result_sets
|
178
|
+
end
|
179
|
+
|
124
180
|
test "save! calls the after_save handler" do
|
125
181
|
x = 41
|
126
182
|
|
@@ -190,8 +246,8 @@ class BulkInsertWorkerTest < ActiveSupport::TestCase
|
|
190
246
|
@insert.add ["Hello", 25, true, @now, @now]
|
191
247
|
@insert.save!
|
192
248
|
|
193
|
-
yo = Testing.
|
194
|
-
hello = Testing.
|
249
|
+
yo = Testing.where(greeting: 'Yo').first
|
250
|
+
hello = Testing.where(greeting: 'Hello').first
|
195
251
|
|
196
252
|
assert_nil yo
|
197
253
|
assert_not_nil hello
|
@@ -206,137 +262,256 @@ class BulkInsertWorkerTest < ActiveSupport::TestCase
|
|
206
262
|
@insert.add ["Hello", 25, true, @now, @now]
|
207
263
|
@insert.save!
|
208
264
|
|
209
|
-
yo = Testing.
|
210
|
-
hello = Testing.
|
265
|
+
yo = Testing.where(greeting: 'Yo').first
|
266
|
+
hello = Testing.where(greeting: 'Hello').first
|
211
267
|
|
212
268
|
assert_nil yo
|
213
269
|
assert_nil hello
|
214
270
|
end
|
215
271
|
|
216
|
-
test "adapter dependent
|
217
|
-
|
218
|
-
|
272
|
+
test "adapter dependent SQLite methods" do
|
273
|
+
connection = Testing.connection
|
274
|
+
stub_connection_if_needed(connection, 'SQLite') do
|
275
|
+
sqlite_worker = BulkInsert::Worker.new(
|
276
|
+
connection,
|
277
|
+
Testing.table_name,
|
278
|
+
'id',
|
279
|
+
%w(greeting age happy created_at updated_at color),
|
280
|
+
500 # batch size
|
281
|
+
)
|
282
|
+
|
283
|
+
assert_equal sqlite_worker.adapter_name, 'SQLite'
|
284
|
+
assert_equal sqlite_worker.insert_sql_statement, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES "
|
219
285
|
|
220
|
-
|
221
|
-
|
286
|
+
sqlite_worker.add ["Yo", 15, false, nil, nil]
|
287
|
+
assert_equal sqlite_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')"
|
288
|
+
end
|
222
289
|
end
|
223
290
|
|
224
|
-
test "adapter dependent
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
291
|
+
test "adapter dependent MySQL methods" do
|
292
|
+
connection = Testing.connection
|
293
|
+
stub_connection_if_needed(connection, 'mysql') do
|
294
|
+
mysql_worker = BulkInsert::Worker.new(
|
295
|
+
connection,
|
296
|
+
Testing.table_name,
|
297
|
+
'id',
|
298
|
+
%w(greeting age happy created_at updated_at color),
|
299
|
+
500, # batch size
|
300
|
+
true # ignore
|
301
|
+
)
|
232
302
|
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
303
|
+
assert_equal mysql_worker.adapter_name, 'mysql'
|
304
|
+
assert_equal (mysql_worker.adapter_name == 'mysql'), true
|
305
|
+
assert_equal mysql_worker.ignore, true
|
306
|
+
assert_equal ((mysql_worker.adapter_name == 'mysql') & mysql_worker.ignore), true
|
237
307
|
|
238
|
-
|
308
|
+
mysql_worker.add ["Yo", 15, false, nil, nil]
|
239
309
|
|
240
|
-
|
310
|
+
assert_statement_adapter mysql_worker, 'BulkInsert::StatementAdapters::MySQLAdapter'
|
311
|
+
assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO `testings` (`greeting`,`age`,`happy`,`created_at`,`updated_at`,`color`) VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse')"
|
312
|
+
end
|
241
313
|
end
|
242
314
|
|
243
315
|
test "adapter dependent mysql methods work for mysql2" do
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
316
|
+
connection = Testing.connection
|
317
|
+
stub_connection_if_needed(connection, 'mysql2') do
|
318
|
+
mysql_worker = BulkInsert::Worker.new(
|
319
|
+
connection,
|
320
|
+
Testing.table_name,
|
321
|
+
'id',
|
322
|
+
%w(greeting age happy created_at updated_at color),
|
323
|
+
500, # batch size
|
324
|
+
true, # ignore
|
325
|
+
true) # update_duplicates
|
326
|
+
|
327
|
+
assert_equal mysql_worker.adapter_name, 'mysql2'
|
328
|
+
assert mysql_worker.ignore
|
329
|
+
|
330
|
+
mysql_worker.add ["Yo", 15, false, nil, nil]
|
331
|
+
|
332
|
+
assert_statement_adapter mysql_worker, 'BulkInsert::StatementAdapters::MySQLAdapter'
|
333
|
+
assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO `testings` (`greeting`,`age`,`happy`,`created_at`,`updated_at`,`color`) VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)"
|
334
|
+
end
|
259
335
|
end
|
260
336
|
|
261
337
|
test "adapter dependent Mysql2Spatial methods" do
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
338
|
+
connection = Testing.connection
|
339
|
+
stub_connection_if_needed(connection, 'mysql2spatial') do
|
340
|
+
mysql_worker = BulkInsert::Worker.new(
|
341
|
+
connection,
|
342
|
+
Testing.table_name,
|
343
|
+
'id',
|
344
|
+
%w(greeting age happy created_at updated_at color),
|
345
|
+
500, # batch size
|
346
|
+
true) # ignore
|
347
|
+
|
348
|
+
assert_equal mysql_worker.adapter_name, 'mysql2spatial'
|
349
|
+
|
350
|
+
mysql_worker.add ["Yo", 15, false, nil, nil]
|
351
|
+
|
352
|
+
assert_statement_adapter mysql_worker, 'BulkInsert::StatementAdapters::MySQLAdapter'
|
353
|
+
assert_equal mysql_worker.compose_insert_query, "INSERT IGNORE INTO `testings` (`greeting`,`age`,`happy`,`created_at`,`updated_at`,`color`) VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse')"
|
354
|
+
end
|
275
355
|
end
|
276
356
|
|
277
357
|
test "adapter dependent postgresql methods" do
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
358
|
+
connection = Testing.connection
|
359
|
+
stub_connection_if_needed(connection, 'PostgreSQL') do
|
360
|
+
pgsql_worker = BulkInsert::Worker.new(
|
361
|
+
connection,
|
362
|
+
Testing.table_name,
|
363
|
+
'id',
|
364
|
+
%w(greeting age happy created_at updated_at color),
|
365
|
+
500, # batch size
|
366
|
+
true, # ignore
|
367
|
+
false, # update duplicates
|
368
|
+
true # return primary keys
|
369
|
+
)
|
370
|
+
|
371
|
+
pgsql_worker.add ["Yo", 15, false, nil, nil]
|
372
|
+
|
373
|
+
assert_statement_adapter pgsql_worker, 'BulkInsert::StatementAdapters::PostgreSQLAdapter'
|
374
|
+
|
375
|
+
if ActiveRecord::VERSION::STRING >= "5.0.0"
|
376
|
+
assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id"
|
377
|
+
else
|
378
|
+
assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,'f',NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id"
|
379
|
+
end
|
380
|
+
end
|
381
|
+
end
|
286
382
|
|
287
|
-
|
383
|
+
test "adapter dependent postgresql methods (no ignore, no update_duplicates)" do
|
384
|
+
connection = Testing.connection
|
385
|
+
stub_connection_if_needed(connection, 'PostgreSQL') do
|
386
|
+
pgsql_worker = BulkInsert::Worker.new(
|
387
|
+
connection,
|
388
|
+
Testing.table_name,
|
389
|
+
'id',
|
390
|
+
%w(greeting age happy created_at updated_at color),
|
391
|
+
500, # batch size
|
392
|
+
false, # ignore
|
393
|
+
false, # update duplicates
|
394
|
+
true # return primary keys
|
395
|
+
)
|
396
|
+
|
397
|
+
pgsql_worker.add ["Yo", 15, false, nil, nil]
|
398
|
+
|
399
|
+
assert_statement_adapter pgsql_worker, 'BulkInsert::StatementAdapters::PostgreSQLAdapter'
|
400
|
+
|
401
|
+
if ActiveRecord::VERSION::STRING >= "5.0.0"
|
402
|
+
assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse') RETURNING id"
|
403
|
+
else
|
404
|
+
assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,'f',NULL,NULL,'chartreuse') RETURNING id"
|
405
|
+
end
|
406
|
+
end
|
288
407
|
end
|
289
408
|
|
290
|
-
test "adapter dependent
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
409
|
+
test "adapter dependent postgresql methods (with update_duplicates)" do
|
410
|
+
connection = Testing.connection
|
411
|
+
stub_connection_if_needed(connection, 'PostgreSQL') do
|
412
|
+
pgsql_worker = BulkInsert::Worker.new(
|
413
|
+
connection,
|
414
|
+
Testing.table_name,
|
415
|
+
'id',
|
416
|
+
%w(greeting age happy created_at updated_at color),
|
417
|
+
500, # batch size
|
418
|
+
false, # ignore
|
419
|
+
%w(greeting age happy), # update duplicates
|
420
|
+
true # return primary keys
|
421
|
+
)
|
422
|
+
pgsql_worker.add ["Yo", 15, false, nil, nil]
|
423
|
+
|
424
|
+
assert_statement_adapter pgsql_worker, 'BulkInsert::StatementAdapters::PostgreSQLAdapter'
|
425
|
+
|
426
|
+
if ActiveRecord::VERSION::STRING >= "5.0.0"
|
427
|
+
assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse') ON CONFLICT(greeting, age, happy) DO UPDATE SET greeting=EXCLUDED.greeting, age=EXCLUDED.age, happy=EXCLUDED.happy, created_at=EXCLUDED.created_at, updated_at=EXCLUDED.updated_at, color=EXCLUDED.color RETURNING id"
|
428
|
+
else
|
429
|
+
assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,'f',NULL,NULL,'chartreuse') ON CONFLICT(greeting, age, happy) DO UPDATE SET greeting=EXCLUDED.greeting, age=EXCLUDED.age, happy=EXCLUDED.happy, created_at=EXCLUDED.created_at, updated_at=EXCLUDED.updated_at, color=EXCLUDED.color RETURNING id"
|
430
|
+
end
|
431
|
+
end
|
432
|
+
end
|
299
433
|
|
300
|
-
|
434
|
+
test "adapter dependent PostGIS methods" do
|
435
|
+
connection = Testing.connection
|
436
|
+
stub_connection_if_needed(connection, 'postgis') do
|
437
|
+
pgsql_worker = BulkInsert::Worker.new(
|
438
|
+
connection,
|
439
|
+
Testing.table_name,
|
440
|
+
'id',
|
441
|
+
%w(greeting age happy created_at updated_at color),
|
442
|
+
500, # batch size
|
443
|
+
true, # ignore
|
444
|
+
false, # update duplicates
|
445
|
+
true # return primary keys
|
446
|
+
)
|
447
|
+
pgsql_worker.add ["Yo", 15, false, nil, nil]
|
448
|
+
|
449
|
+
assert_statement_adapter pgsql_worker, 'BulkInsert::StatementAdapters::PostgreSQLAdapter'
|
450
|
+
|
451
|
+
if ActiveRecord::VERSION::STRING >= "5.0.0"
|
452
|
+
assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id"
|
453
|
+
else
|
454
|
+
assert_equal pgsql_worker.compose_insert_query, "INSERT INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,'f',NULL,NULL,'chartreuse') ON CONFLICT DO NOTHING RETURNING id"
|
455
|
+
end
|
456
|
+
end
|
301
457
|
end
|
302
458
|
|
303
459
|
test "adapter dependent sqlite3 methods (with lowercase adapter name)" do
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
460
|
+
connection = Testing.connection
|
461
|
+
stub_connection_if_needed(connection, 'sqlite3') do
|
462
|
+
sqlite_worker = BulkInsert::Worker.new(
|
463
|
+
Testing.connection,
|
464
|
+
Testing.table_name,
|
465
|
+
'id',
|
466
|
+
%w(greeting age happy created_at updated_at color),
|
467
|
+
500, # batch size
|
468
|
+
true) # ignore
|
469
|
+
sqlite_worker.adapter_name = 'sqlite3'
|
470
|
+
sqlite_worker.add ["Yo", 15, false, nil, nil]
|
471
|
+
|
472
|
+
assert_statement_adapter sqlite_worker, 'BulkInsert::StatementAdapters::SQLiteAdapter'
|
473
|
+
assert_equal sqlite_worker.compose_insert_query, "INSERT OR IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')"
|
474
|
+
end
|
314
475
|
end
|
315
476
|
|
316
477
|
test "adapter dependent sqlite3 methods (with stylecase adapter name)" do
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
478
|
+
connection = Testing.connection
|
479
|
+
stub_connection_if_needed(connection, 'SQLite') do
|
480
|
+
sqlite_worker = BulkInsert::Worker.new(
|
481
|
+
connection,
|
482
|
+
Testing.table_name,
|
483
|
+
'id',
|
484
|
+
%w(greeting age happy created_at updated_at color),
|
485
|
+
500, # batch size
|
486
|
+
true) # ignore
|
487
|
+
sqlite_worker.adapter_name = 'SQLite'
|
488
|
+
sqlite_worker.add ["Yo", 15, false, nil, nil]
|
489
|
+
|
490
|
+
assert_statement_adapter sqlite_worker, 'BulkInsert::StatementAdapters::SQLiteAdapter'
|
491
|
+
assert_equal sqlite_worker.compose_insert_query, "INSERT OR IGNORE INTO \"testings\" (\"greeting\",\"age\",\"happy\",\"created_at\",\"updated_at\",\"color\") VALUES ('Yo',15,0,NULL,NULL,'chartreuse')"
|
492
|
+
end
|
327
493
|
end
|
328
494
|
|
329
495
|
test "mysql adapter can update duplicates" do
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
496
|
+
connection = Testing.connection
|
497
|
+
stub_connection_if_needed(connection, 'mysql') do
|
498
|
+
mysql_worker = BulkInsert::Worker.new(
|
499
|
+
connection,
|
500
|
+
Testing.table_name,
|
501
|
+
'id',
|
502
|
+
%w(greeting age happy created_at updated_at color),
|
503
|
+
500, # batch size
|
504
|
+
false, # ignore
|
505
|
+
true # update_duplicates
|
506
|
+
)
|
507
|
+
mysql_worker.add ["Yo", 15, false, nil, nil]
|
508
|
+
|
509
|
+
assert_statement_adapter mysql_worker, 'BulkInsert::StatementAdapters::MySQLAdapter'
|
510
|
+
assert_equal mysql_worker.compose_insert_query, "INSERT INTO `testings` (`greeting`,`age`,`happy`,`created_at`,`updated_at`,`color`) VALUES ('Yo',15,FALSE,NULL,NULL,'chartreuse') ON DUPLICATE KEY UPDATE `greeting`=VALUES(`greeting`), `age`=VALUES(`age`), `happy`=VALUES(`happy`), `created_at`=VALUES(`created_at`), `updated_at`=VALUES(`updated_at`), `color`=VALUES(`color`)"
|
511
|
+
end
|
512
|
+
end
|
339
513
|
|
340
|
-
|
514
|
+
def assert_statement_adapter(worker, adapter_name)
|
515
|
+
assert_equal worker.instance_variable_get(:@statement_adapter).class.to_s, adapter_name
|
341
516
|
end
|
342
517
|
end
|