upsert 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +17 -0
- data/Gemfile +4 -0
- data/LICENSE +22 -0
- data/README.md +70 -0
- data/Rakefile +25 -0
- data/lib/upsert.rb +41 -0
- data/lib/upsert/buffer.rb +58 -0
- data/lib/upsert/buffer/mysql2_client.rb +47 -0
- data/lib/upsert/buffer/pg_connection.rb +94 -0
- data/lib/upsert/buffer/pg_connection/column_definition.rb +34 -0
- data/lib/upsert/buffer/sqlite3_database.rb +44 -0
- data/lib/upsert/quoter.rb +15 -0
- data/lib/upsert/row.rb +41 -0
- data/lib/upsert/version.rb +3 -0
- data/test/helper.rb +54 -0
- data/test/shared_examples.rb +94 -0
- data/test/test_mysql2.rb +28 -0
- data/test/test_pg.rb +29 -0
- data/test/test_sqlite.rb +24 -0
- data/test/test_upsert.rb +7 -0
- data/upsert.gemspec +26 -0
- metadata +204 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Seamus Abshere
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
# Upsert
|
2
|
+
|
3
|
+
Finally, all those SQL MERGE tricks codified.
|
4
|
+
|
5
|
+
## Supported databases
|
6
|
+
|
7
|
+
### MySQL
|
8
|
+
|
9
|
+
# http://dev.mysql.com/doc/refman/5.0/en/insert-on-duplicate.html
|
10
|
+
INSERT INTO table (a,b,c) VALUES (1,2,3)
|
11
|
+
ON DUPLICATE KEY UPDATE c=c+1;
|
12
|
+
|
13
|
+
### PostgreSQL
|
14
|
+
|
15
|
+
#### Used
|
16
|
+
|
17
|
+
# http://www.postgresql.org/docs/current/interactive/plpgsql-control-structures.html#PLPGSQL-ERROR-TRAPPING
|
18
|
+
CREATE TABLE db (a INT PRIMARY KEY, b TEXT);
|
19
|
+
CREATE FUNCTION merge_db(key INT, data TEXT) RETURNS VOID AS
|
20
|
+
$$
|
21
|
+
BEGIN
|
22
|
+
LOOP
|
23
|
+
-- first try to update the key
|
24
|
+
UPDATE db SET b = data WHERE a = key;
|
25
|
+
IF found THEN
|
26
|
+
RETURN;
|
27
|
+
END IF;
|
28
|
+
-- not there, so try to insert the key
|
29
|
+
-- if someone else inserts the same key concurrently,
|
30
|
+
-- we could get a unique-key failure
|
31
|
+
BEGIN
|
32
|
+
INSERT INTO db(a,b) VALUES (key, data);
|
33
|
+
RETURN;
|
34
|
+
EXCEPTION WHEN unique_violation THEN
|
35
|
+
-- Do nothing, and loop to try the UPDATE again.
|
36
|
+
END;
|
37
|
+
END LOOP;
|
38
|
+
END;
|
39
|
+
$$
|
40
|
+
LANGUAGE plpgsql;
|
41
|
+
SELECT merge_db(1, 'david');
|
42
|
+
SELECT merge_db(1, 'dennis');
|
43
|
+
|
44
|
+
#### Alternatives (not used)
|
45
|
+
|
46
|
+
# http://stackoverflow.com/questions/1109061/insert-on-duplicate-update-postgresql
|
47
|
+
UPDATE table SET field='C', field2='Z' WHERE id=3;
|
48
|
+
INSERT INTO table (id, field, field2)
|
49
|
+
SELECT 3, 'C', 'Z'
|
50
|
+
WHERE NOT EXISTS (SELECT 1 FROM table WHERE id=3);
|
51
|
+
|
52
|
+
# http://stackoverflow.com/questions/5269590/why-doesnt-this-rule-prevent-duplicate-key-violations
|
53
|
+
BEGIN;
|
54
|
+
CREATE TEMP TABLE stage_data(key_column, data_columns...) ON COMMIT DROP;
|
55
|
+
\copy stage_data from data.csv with csv header
|
56
|
+
-- prevent any other updates while we are merging input (omit this if you don't need it)
|
57
|
+
LOCK target_data IN SHARE ROW EXCLUSIVE MODE;
|
58
|
+
-- insert into target table
|
59
|
+
INSERT INTO target_data(key_column, data_columns...)
|
60
|
+
SELECT key_column, data_columns...
|
61
|
+
FROM stage_data
|
62
|
+
WHERE NOT EXISTS (SELECT 1 FROM target_data
|
63
|
+
WHERE target_data.key_column = stage_data.key_column)
|
64
|
+
END;
|
65
|
+
|
66
|
+
### Sqlite
|
67
|
+
|
68
|
+
# http://stackoverflow.com/questions/2717590/sqlite-upsert-on-duplicate-key-update
|
69
|
+
INSERT OR IGNORE INTO visits VALUES ($ip, 0);
|
70
|
+
UPDATE visits SET hits = hits + 1 WHERE ip LIKE $ip;
|
data/Rakefile
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
|
4
|
+
require 'rake'
|
5
|
+
require 'rake/testtask'
|
6
|
+
Rake::TestTask.new(:_test) do |test|
|
7
|
+
test.libs << 'lib' << 'test'
|
8
|
+
test.pattern = 'test/**/test_*.rb'
|
9
|
+
test.verbose = true
|
10
|
+
end
|
11
|
+
|
12
|
+
task :test_each_db_adapter do
|
13
|
+
%w{ mysql2 sqlite pg }.each do |database|
|
14
|
+
puts
|
15
|
+
puts "#{'*'*10} Running #{database} tests"
|
16
|
+
puts
|
17
|
+
puts `rake _test TEST=test/test_#{database}.rb`
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
task :default => :test_each_db_adapter
|
22
|
+
task :test => :test_each_db_adapter
|
23
|
+
|
24
|
+
require 'yard'
|
25
|
+
YARD::Rake::YardocTask.new
|
data/lib/upsert.rb
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'upsert/version'
|
2
|
+
require 'upsert/buffer'
|
3
|
+
require 'upsert/quoter'
|
4
|
+
require 'upsert/row'
|
5
|
+
require 'upsert/buffer/mysql2_client'
|
6
|
+
require 'upsert/buffer/pg_connection'
|
7
|
+
require 'upsert/buffer/sqlite3_database'
|
8
|
+
|
9
|
+
class Upsert
|
10
|
+
INFINITY = 1.0/0
|
11
|
+
SINGLE_QUOTE = %{'}
|
12
|
+
DOUBLE_QUOTE = %{"}
|
13
|
+
BACKTICK = %{`}
|
14
|
+
|
15
|
+
attr_reader :buffer
|
16
|
+
|
17
|
+
def initialize(connection, table_name)
|
18
|
+
@multi_mutex = Mutex.new
|
19
|
+
@buffer = Buffer.for connection, table_name
|
20
|
+
end
|
21
|
+
|
22
|
+
def row(selector, document)
|
23
|
+
buffer.add selector, document
|
24
|
+
end
|
25
|
+
|
26
|
+
def cleanup
|
27
|
+
buffer.cleanup
|
28
|
+
end
|
29
|
+
|
30
|
+
def multi(&blk)
|
31
|
+
@multi_mutex.synchronize do
|
32
|
+
begin
|
33
|
+
buffer.async = true
|
34
|
+
instance_eval(&blk)
|
35
|
+
buffer.cleanup
|
36
|
+
ensure
|
37
|
+
buffer.async = nil
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
class Upsert
|
2
|
+
class Buffer
|
3
|
+
class << self
|
4
|
+
def for(connection, table_name)
|
5
|
+
const_get(connection.class.name.gsub(/\W+/, '_')).new connection, table_name
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
attr_reader :connection
|
10
|
+
attr_reader :table_name
|
11
|
+
attr_reader :rows
|
12
|
+
attr_writer :async
|
13
|
+
|
14
|
+
def initialize(connection, table_name)
|
15
|
+
@connection = connection
|
16
|
+
@table_name = table_name
|
17
|
+
@rows = []
|
18
|
+
end
|
19
|
+
|
20
|
+
def async?
|
21
|
+
!!@async
|
22
|
+
end
|
23
|
+
|
24
|
+
def add(selector, document)
|
25
|
+
rows << Row.new(selector, document)
|
26
|
+
if sql = chunk
|
27
|
+
execute sql
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def clear
|
32
|
+
while sql = chunk
|
33
|
+
execute sql
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def chunk
|
38
|
+
return if rows.empty?
|
39
|
+
targets = []
|
40
|
+
sql = nil
|
41
|
+
begin
|
42
|
+
targets << rows.pop
|
43
|
+
last_sql = sql
|
44
|
+
sql = compose(targets)
|
45
|
+
end until rows.empty? or targets.length >= max_targets or sql.length > max_length
|
46
|
+
if sql.length > max_length
|
47
|
+
raise if last_sql.nil?
|
48
|
+
sql = last_sql
|
49
|
+
rows << targets.pop
|
50
|
+
end
|
51
|
+
sql
|
52
|
+
end
|
53
|
+
|
54
|
+
def cleanup
|
55
|
+
clear
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
class Upsert
|
2
|
+
class Buffer
|
3
|
+
class Mysql2_Client < Buffer
|
4
|
+
def compose(targets)
|
5
|
+
columns = targets.first.columns
|
6
|
+
row_inserts = targets.map { |row| row.inserts }
|
7
|
+
column_tautologies = columns.map do |k|
|
8
|
+
[ quote_ident(k), "VALUES(#{quote_ident(k)})" ].join('=')
|
9
|
+
end
|
10
|
+
sql = <<-EOS
|
11
|
+
INSERT INTO "#{table_name}" (#{quote_idents(columns)}) VALUES (#{row_inserts.map { |row_insert| quote_values(row_insert) }.join('),(') })
|
12
|
+
ON DUPLICATE KEY UPDATE #{column_tautologies.join(',')};
|
13
|
+
EOS
|
14
|
+
sql
|
15
|
+
end
|
16
|
+
|
17
|
+
def execute(sql)
|
18
|
+
connection.query sql
|
19
|
+
end
|
20
|
+
|
21
|
+
def max_targets
|
22
|
+
INFINITY
|
23
|
+
end
|
24
|
+
|
25
|
+
def max_length
|
26
|
+
@max_length ||= connection.query("SHOW VARIABLES LIKE 'max_allowed_packet'", :as => :hash).first['Value'].to_i
|
27
|
+
end
|
28
|
+
|
29
|
+
include Quoter
|
30
|
+
|
31
|
+
def quote_value(v)
|
32
|
+
case v
|
33
|
+
when NilClass
|
34
|
+
'NULL'
|
35
|
+
when String, Symbol
|
36
|
+
SINGLE_QUOTE + connection.escape(v.to_s) + SINGLE_QUOTE
|
37
|
+
else
|
38
|
+
v
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def quote_ident(k)
|
43
|
+
BACKTICK + connection.escape(k.to_s) + BACKTICK
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
require 'upsert/buffer/pg_connection/column_definition'
|
2
|
+
|
3
|
+
class Upsert
|
4
|
+
class Buffer
|
5
|
+
class PG_Connection < Buffer
|
6
|
+
attr_reader :db_function_name
|
7
|
+
|
8
|
+
def compose(targets)
|
9
|
+
target = targets.first
|
10
|
+
unless created_db_function?
|
11
|
+
create_db_function target
|
12
|
+
end
|
13
|
+
hsh = target.to_hash
|
14
|
+
ordered_args = column_definitions.map do |c|
|
15
|
+
if hsh.has_key? c.name
|
16
|
+
hsh[c.name]
|
17
|
+
else
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
end
|
21
|
+
%{ SELECT #{db_function_name}(#{quote_values(ordered_args)}) }
|
22
|
+
end
|
23
|
+
|
24
|
+
def execute(sql)
|
25
|
+
connection.exec sql
|
26
|
+
end
|
27
|
+
|
28
|
+
def max_length
|
29
|
+
INFINITY
|
30
|
+
end
|
31
|
+
|
32
|
+
def max_targets
|
33
|
+
1
|
34
|
+
end
|
35
|
+
|
36
|
+
include Quoter
|
37
|
+
|
38
|
+
def quote_ident(k)
|
39
|
+
SINGLE_QUOTE + connection.quote_ident(k) + SINGLE_QUOTE
|
40
|
+
end
|
41
|
+
|
42
|
+
# FIXME escape_bytea with (v, k = nil)
|
43
|
+
def quote_value(v)
|
44
|
+
case v
|
45
|
+
when NilClass
|
46
|
+
'NULL'
|
47
|
+
when String, Symbol
|
48
|
+
SINGLE_QUOTE + connection.escape_string(v.to_s) + SINGLE_QUOTE
|
49
|
+
else
|
50
|
+
v
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def column_definitions
|
55
|
+
@column_definitions ||= ColumnDefinition.all(connection, table_name)
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
|
60
|
+
def created_db_function?
|
61
|
+
!!@created_db_function_query
|
62
|
+
end
|
63
|
+
|
64
|
+
def create_db_function(example_row)
|
65
|
+
@db_function_name = "pg_temp.merge_#{table_name}_#{Kernel.rand(1e11)}"
|
66
|
+
execute <<-EOS
|
67
|
+
CREATE FUNCTION #{db_function_name}(#{column_definitions.map { |c| "#{c.name}_input #{c.sql_type} DEFAULT #{c.default || 'NULL'}" }.join(',') }) RETURNS VOID AS
|
68
|
+
$$
|
69
|
+
BEGIN
|
70
|
+
LOOP
|
71
|
+
-- first try to update the key
|
72
|
+
UPDATE #{table_name} SET #{column_definitions.map { |c| "#{c.name} = #{c.name}_input" }.join(',')} WHERE #{example_row.selector.keys.map { |k| "#{k} = #{k}_input" }.join(' AND ') };
|
73
|
+
IF found THEN
|
74
|
+
RETURN;
|
75
|
+
END IF;
|
76
|
+
-- not there, so try to insert the key
|
77
|
+
-- if someone else inserts the same key concurrently,
|
78
|
+
-- we could get a unique-key failure
|
79
|
+
BEGIN
|
80
|
+
INSERT INTO #{table_name}(#{column_definitions.map { |c| c.name }.join(',')}) VALUES (#{column_definitions.map { |c| "#{c.name}_input" }.join(',')});
|
81
|
+
RETURN;
|
82
|
+
EXCEPTION WHEN unique_violation THEN
|
83
|
+
-- Do nothing, and loop to try the UPDATE again.
|
84
|
+
END;
|
85
|
+
END LOOP;
|
86
|
+
END;
|
87
|
+
$$
|
88
|
+
LANGUAGE plpgsql;
|
89
|
+
EOS
|
90
|
+
@created_db_function_query = true
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
class Upsert
|
2
|
+
class Buffer
|
3
|
+
class PG_Connection < Buffer
|
4
|
+
class ColumnDefinition
|
5
|
+
class << self
|
6
|
+
def all(connection, table_name)
|
7
|
+
# activerecord-3.2.5/lib/active_record/connection_adapters/postgresql_adapter.rb#column_definitions
|
8
|
+
res = connection.exec <<-EOS
|
9
|
+
SELECT a.attname AS name, format_type(a.atttypid, a.atttypmod) AS sql_type, d.adsrc AS default
|
10
|
+
FROM pg_attribute a LEFT JOIN pg_attrdef d
|
11
|
+
ON a.attrelid = d.adrelid AND a.attnum = d.adnum
|
12
|
+
WHERE a.attrelid = '#{connection.quote_ident(table_name.to_s)}'::regclass
|
13
|
+
AND a.attnum > 0 AND NOT a.attisdropped
|
14
|
+
ORDER BY a.attnum
|
15
|
+
EOS
|
16
|
+
res.map do |row|
|
17
|
+
new row['name'], row['sql_type'], row['default']
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
attr_reader :name
|
23
|
+
attr_reader :sql_type
|
24
|
+
attr_reader :default
|
25
|
+
|
26
|
+
def initialize(name, sql_type, default)
|
27
|
+
@name = name
|
28
|
+
@sql_type = sql_type
|
29
|
+
@default = default
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
class Upsert
|
2
|
+
class Buffer
|
3
|
+
class SQLite3_Database < Buffer
|
4
|
+
def compose(targets)
|
5
|
+
target = targets.first
|
6
|
+
parts = []
|
7
|
+
parts << %{ INSERT OR IGNORE INTO "#{table_name}" (#{quote_idents(target.columns)}) VALUES (#{quote_values(target.inserts)}) }
|
8
|
+
if target.updates.length > 0
|
9
|
+
parts << %{ UPDATE "#{table_name}" SET #{quote_pairs(target.updates)} WHERE #{quote_pairs(target.selector)} }
|
10
|
+
end
|
11
|
+
parts.join(';')
|
12
|
+
end
|
13
|
+
|
14
|
+
def execute(sql)
|
15
|
+
connection.execute_batch sql
|
16
|
+
end
|
17
|
+
|
18
|
+
def max_targets
|
19
|
+
1
|
20
|
+
end
|
21
|
+
|
22
|
+
def max_length
|
23
|
+
INFINITY
|
24
|
+
end
|
25
|
+
|
26
|
+
include Quoter
|
27
|
+
|
28
|
+
def quote_value(v)
|
29
|
+
case v
|
30
|
+
when NilClass
|
31
|
+
'NULL'
|
32
|
+
when String, Symbol
|
33
|
+
SINGLE_QUOTE + SQLite3::Database.quote(v.to_s) + SINGLE_QUOTE
|
34
|
+
else
|
35
|
+
v
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def quote_ident(k)
|
40
|
+
DOUBLE_QUOTE + SQLite3::Database.quote(k.to_s) + DOUBLE_QUOTE
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class Upsert
|
2
|
+
module Quoter
|
3
|
+
def quote_idents(idents)
|
4
|
+
idents.map { |k| quote_ident(k) }.join(',')
|
5
|
+
end
|
6
|
+
|
7
|
+
def quote_values(values)
|
8
|
+
values.map { |v| quote_value(v) }.join(',')
|
9
|
+
end
|
10
|
+
|
11
|
+
def quote_pairs(pairs)
|
12
|
+
pairs.map { |k, v| [quote_ident(k),quote_value(v)].join('=') }.join(',')
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
data/lib/upsert/row.rb
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
class Upsert
|
2
|
+
class Row
|
3
|
+
attr_reader :selector
|
4
|
+
attr_reader :document
|
5
|
+
|
6
|
+
def initialize(selector, document)
|
7
|
+
@selector = selector
|
8
|
+
@document = document
|
9
|
+
end
|
10
|
+
|
11
|
+
def columns
|
12
|
+
@columns ||= (selector.keys+document.keys).uniq
|
13
|
+
end
|
14
|
+
|
15
|
+
def pairs
|
16
|
+
@pairs ||= columns.map do |k|
|
17
|
+
value = if selector.has_key?(k)
|
18
|
+
selector[k]
|
19
|
+
else
|
20
|
+
document[k]
|
21
|
+
end
|
22
|
+
[ k, value ]
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def inserts
|
27
|
+
@inserts ||= pairs.map { |_, v| v }
|
28
|
+
end
|
29
|
+
|
30
|
+
def updates
|
31
|
+
@updates ||= pairs.reject { |k, _| selector.has_key?(k) }
|
32
|
+
end
|
33
|
+
|
34
|
+
def to_hash
|
35
|
+
@to_hash ||= pairs.inject({}) do |memo, (k, v)|
|
36
|
+
memo[k.to_s] = v
|
37
|
+
memo
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler/setup'
|
3
|
+
require 'minitest/spec'
|
4
|
+
require 'minitest/autorun'
|
5
|
+
require 'minitest/reporters'
|
6
|
+
MiniTest::Unit.runner = MiniTest::SuiteRunner.new
|
7
|
+
MiniTest::Unit.runner.reporters << MiniTest::Reporters::SpecReporter.new
|
8
|
+
|
9
|
+
require 'active_record'
|
10
|
+
require 'active_record_inline_schema'
|
11
|
+
|
12
|
+
# require 'logger'
|
13
|
+
# ActiveRecord::Base.logger = Logger.new($stdout)
|
14
|
+
# ActiveRecord::Base.logger.level = Logger::DEBUG
|
15
|
+
|
16
|
+
class Pet < ActiveRecord::Base
|
17
|
+
self.primary_key = 'name'
|
18
|
+
col :name
|
19
|
+
col :gender
|
20
|
+
end
|
21
|
+
|
22
|
+
require 'upsert'
|
23
|
+
|
24
|
+
MiniTest::Spec.class_eval do
|
25
|
+
def self.shared_examples
|
26
|
+
@shared_examples ||= {}
|
27
|
+
end
|
28
|
+
|
29
|
+
def assert_creates(model, expected_records)
|
30
|
+
expected_records.each do |conditions|
|
31
|
+
model.where(conditions).count.must_equal 0
|
32
|
+
end
|
33
|
+
yield
|
34
|
+
expected_records.each do |conditions|
|
35
|
+
model.where(conditions).count.must_equal 1
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
module MiniTest::Spec::SharedExamples
|
42
|
+
def shared_examples_for(desc, &block)
|
43
|
+
MiniTest::Spec.shared_examples[desc] = block
|
44
|
+
end
|
45
|
+
|
46
|
+
def it_behaves_like(desc)
|
47
|
+
self.instance_eval do
|
48
|
+
MiniTest::Spec.shared_examples[desc].call
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
Object.class_eval { include(MiniTest::Spec::SharedExamples) }
|
54
|
+
require 'shared_examples'
|
@@ -0,0 +1,94 @@
|
|
1
|
+
shared_examples_for :database do
|
2
|
+
describe :row do
|
3
|
+
it "works for a single row (base case)" do
|
4
|
+
upsert = Upsert.new connection, :pets
|
5
|
+
assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
|
6
|
+
upsert.row({:name => 'Jerry'}, {:gender => 'male'})
|
7
|
+
end
|
8
|
+
end
|
9
|
+
it "works for a single row (not changing anything)" do
|
10
|
+
upsert = Upsert.new connection, :pets
|
11
|
+
assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
|
12
|
+
upsert.row({:name => 'Jerry'}, {:gender => 'male'})
|
13
|
+
upsert.row({:name => 'Jerry'}, {:gender => 'male'})
|
14
|
+
end
|
15
|
+
end
|
16
|
+
it "works for a single row (changing something)" do
|
17
|
+
upsert = Upsert.new connection, :pets
|
18
|
+
assert_creates(Pet, [{:name => 'Jerry', :gender => 'neutered'}]) do
|
19
|
+
upsert.row({:name => 'Jerry'}, {:gender => 'male'})
|
20
|
+
upsert.row({:name => 'Jerry'}, {:gender => 'neutered'})
|
21
|
+
end
|
22
|
+
Pet.where(:gender => 'male').count.must_equal 0
|
23
|
+
end
|
24
|
+
|
25
|
+
it "works for a single row with implicit nulls" do
|
26
|
+
upsert = Upsert.new connection, :pets
|
27
|
+
assert_creates(Pet, [{:name => 'Inky', :gender => nil}]) do
|
28
|
+
upsert.row({:name => 'Inky'}, {})
|
29
|
+
upsert.row({:name => 'Inky'}, {})
|
30
|
+
end
|
31
|
+
end
|
32
|
+
it "works for a single row with explicit nulls" do
|
33
|
+
upsert = Upsert.new connection, :pets
|
34
|
+
assert_creates(Pet, [{:name => 'Inky', :gender => nil}]) do
|
35
|
+
upsert.row({:name => 'Inky'}, {:gender => nil})
|
36
|
+
upsert.row({:name => 'Inky'}, {:gender => nil})
|
37
|
+
end
|
38
|
+
end
|
39
|
+
# it "works for a single row upserted many times" do
|
40
|
+
# assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
|
41
|
+
# ts = (0..5).map do
|
42
|
+
# Thread.new do
|
43
|
+
# upsert = Upsert.new new_connection, :pets
|
44
|
+
# upsert.row({:name => 'Jerry'}, {:gender => 'male'})
|
45
|
+
# end
|
46
|
+
# end
|
47
|
+
# ts.each { |t| t.join }
|
48
|
+
# end
|
49
|
+
# end
|
50
|
+
end
|
51
|
+
describe :multi do
|
52
|
+
it "works for multiple rows (base case)" do
|
53
|
+
upsert = Upsert.new connection, :pets
|
54
|
+
assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
|
55
|
+
upsert.multi do
|
56
|
+
row({:name => 'Jerry'}, :gender => 'male')
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
it "works for multiple rows (not changing anything)" do
|
61
|
+
upsert = Upsert.new connection, :pets
|
62
|
+
assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
|
63
|
+
upsert.multi do
|
64
|
+
row({:name => 'Jerry'}, :gender => 'male')
|
65
|
+
row({:name => 'Jerry'}, :gender => 'male')
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
it "works for multiple rows (changing something)" do
|
70
|
+
upsert = Upsert.new connection, :pets
|
71
|
+
assert_creates(Pet, [{:name => 'Jerry', :gender => 'neutered'}]) do
|
72
|
+
upsert.multi do
|
73
|
+
row({:name => 'Jerry'}, :gender => 'male')
|
74
|
+
row({:name => 'Jerry'}, :gender => 'neutered')
|
75
|
+
end
|
76
|
+
end
|
77
|
+
Pet.where(:gender => 'male').count.must_equal 0
|
78
|
+
end
|
79
|
+
# it "works for multiple rows upserted many times" do
|
80
|
+
# assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
|
81
|
+
# ts = (0..5).map do
|
82
|
+
# Thread.new do
|
83
|
+
# upsert = Upsert.new new_connection, :pets
|
84
|
+
# upsert.multi do
|
85
|
+
# row({:name => 'Jerry'}, :gender => 'male')
|
86
|
+
# row({:name => 'Jerry'}, :gender => 'male')
|
87
|
+
# end
|
88
|
+
# end
|
89
|
+
# end
|
90
|
+
# ts.each { |t| t.join }
|
91
|
+
# end
|
92
|
+
# end
|
93
|
+
end
|
94
|
+
end
|
data/test/test_mysql2.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'mysql2'
|
3
|
+
|
4
|
+
system %{ mysql -u root -ppassword -e "DROP DATABASE IF EXISTS test_upsert; CREATE DATABASE test_upsert CHARSET utf8" }
|
5
|
+
ActiveRecord::Base.establish_connection :adapter => 'mysql2', :username => 'root', :password => 'password', :database => 'test_upsert'
|
6
|
+
|
7
|
+
describe "upserting on mysql2" do
|
8
|
+
before do
|
9
|
+
ActiveRecord::Base.connection.drop_table Pet.table_name rescue nil
|
10
|
+
Pet.auto_upgrade!
|
11
|
+
@opened_connections = []
|
12
|
+
@connection = new_connection
|
13
|
+
end
|
14
|
+
after do
|
15
|
+
@opened_connections.each { |c| c.close }
|
16
|
+
end
|
17
|
+
def new_connection
|
18
|
+
c = Mysql2::Client.new(:username => 'root', :password => 'password', :database => 'test_upsert')
|
19
|
+
@opened_connections << c
|
20
|
+
c
|
21
|
+
end
|
22
|
+
def connection
|
23
|
+
@connection
|
24
|
+
end
|
25
|
+
|
26
|
+
it_behaves_like :database
|
27
|
+
|
28
|
+
end
|
data/test/test_pg.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'pg'
|
3
|
+
|
4
|
+
system %{ dropdb test_upsert }
|
5
|
+
system %{ createdb test_upsert }
|
6
|
+
ActiveRecord::Base.establish_connection :adapter => 'postgresql', :database => 'test_upsert'
|
7
|
+
|
8
|
+
describe "upserting on postgresql" do
|
9
|
+
before do
|
10
|
+
ActiveRecord::Base.connection.drop_table Pet.table_name rescue nil
|
11
|
+
Pet.auto_upgrade!
|
12
|
+
@opened_connections = []
|
13
|
+
@connection = new_connection
|
14
|
+
end
|
15
|
+
after do
|
16
|
+
@opened_connections.each { |c| c.finish }
|
17
|
+
end
|
18
|
+
def new_connection
|
19
|
+
c = PG.connect(:dbname => 'test_upsert')
|
20
|
+
@opened_connections << c
|
21
|
+
c
|
22
|
+
end
|
23
|
+
def connection
|
24
|
+
@connection
|
25
|
+
end
|
26
|
+
|
27
|
+
it_behaves_like :database
|
28
|
+
|
29
|
+
end
|
data/test/test_sqlite.rb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'sqlite3'
|
3
|
+
|
4
|
+
db_path = File.expand_path('../../tmp/test.sqlite3', __FILE__)
|
5
|
+
FileUtils.mkdir_p File.dirname(db_path)
|
6
|
+
FileUtils.rm_f db_path
|
7
|
+
ActiveRecord::Base.establish_connection :adapter => 'sqlite3', :database => db_path
|
8
|
+
|
9
|
+
describe "upserting on sqlite" do
|
10
|
+
before do
|
11
|
+
ActiveRecord::Base.connection.drop_table Pet.table_name rescue nil
|
12
|
+
Pet.auto_upgrade!
|
13
|
+
@connection = new_connection
|
14
|
+
end
|
15
|
+
def new_connection
|
16
|
+
db_path = File.expand_path('../../tmp/test.sqlite3', __FILE__)
|
17
|
+
SQLite3::Database.open(db_path)
|
18
|
+
end
|
19
|
+
def connection
|
20
|
+
@connection
|
21
|
+
end
|
22
|
+
|
23
|
+
it_behaves_like :database
|
24
|
+
end
|
data/test/test_upsert.rb
ADDED
data/upsert.gemspec
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/upsert/version', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.authors = ["Seamus Abshere"]
|
6
|
+
gem.email = ["seamus@abshere.net"]
|
7
|
+
gem.description = %q{Upsert for MySQL, PostgreSQL, and SQLite. Codifies various SQL MERGE tricks like MySQL's ON DUPLICATE KEY UPDATE, PostgreSQL's CREATE FUNCTION merge_db, and SQLite's INSERT OR IGNORE.}
|
8
|
+
gem.summary = %q{Upsert for MySQL, PostgreSQL, and SQLite. Finally, all those SQL MERGE tricks codified.}
|
9
|
+
gem.homepage = "https://github.com/seamusabshere/upsert"
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
|
+
gem.name = "upsert"
|
15
|
+
gem.require_paths = ["lib"]
|
16
|
+
gem.version = Upsert::VERSION
|
17
|
+
|
18
|
+
gem.add_development_dependency 'sqlite3'
|
19
|
+
gem.add_development_dependency 'mysql2'
|
20
|
+
gem.add_development_dependency 'pg'
|
21
|
+
gem.add_development_dependency 'activerecord' # testing only
|
22
|
+
gem.add_development_dependency 'active_record_inline_schema'
|
23
|
+
gem.add_development_dependency 'minitest'
|
24
|
+
gem.add_development_dependency 'minitest-reporters'
|
25
|
+
gem.add_development_dependency 'yard'
|
26
|
+
end
|
metadata
ADDED
@@ -0,0 +1,204 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: upsert
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Seamus Abshere
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-06-13 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: sqlite3
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: mysql2
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: pg
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: activerecord
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: active_record_inline_schema
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
type: :development
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: minitest
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
102
|
+
type: :development
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
- !ruby/object:Gem::Dependency
|
111
|
+
name: minitest-reporters
|
112
|
+
requirement: !ruby/object:Gem::Requirement
|
113
|
+
none: false
|
114
|
+
requirements:
|
115
|
+
- - ! '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
none: false
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: yard
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '0'
|
134
|
+
type: :development
|
135
|
+
prerelease: false
|
136
|
+
version_requirements: !ruby/object:Gem::Requirement
|
137
|
+
none: false
|
138
|
+
requirements:
|
139
|
+
- - ! '>='
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '0'
|
142
|
+
description: Upsert for MySQL, PostgreSQL, and SQLite. Codifies various SQL MERGE
|
143
|
+
tricks like MySQL's ON DUPLICATE KEY UPDATE, PostgreSQL's CREATE FUNCTION merge_db,
|
144
|
+
and SQLite's INSERT OR IGNORE.
|
145
|
+
email:
|
146
|
+
- seamus@abshere.net
|
147
|
+
executables: []
|
148
|
+
extensions: []
|
149
|
+
extra_rdoc_files: []
|
150
|
+
files:
|
151
|
+
- .gitignore
|
152
|
+
- Gemfile
|
153
|
+
- LICENSE
|
154
|
+
- README.md
|
155
|
+
- Rakefile
|
156
|
+
- lib/upsert.rb
|
157
|
+
- lib/upsert/buffer.rb
|
158
|
+
- lib/upsert/buffer/mysql2_client.rb
|
159
|
+
- lib/upsert/buffer/pg_connection.rb
|
160
|
+
- lib/upsert/buffer/pg_connection/column_definition.rb
|
161
|
+
- lib/upsert/buffer/sqlite3_database.rb
|
162
|
+
- lib/upsert/quoter.rb
|
163
|
+
- lib/upsert/row.rb
|
164
|
+
- lib/upsert/version.rb
|
165
|
+
- test/helper.rb
|
166
|
+
- test/shared_examples.rb
|
167
|
+
- test/test_mysql2.rb
|
168
|
+
- test/test_pg.rb
|
169
|
+
- test/test_sqlite.rb
|
170
|
+
- test/test_upsert.rb
|
171
|
+
- upsert.gemspec
|
172
|
+
homepage: https://github.com/seamusabshere/upsert
|
173
|
+
licenses: []
|
174
|
+
post_install_message:
|
175
|
+
rdoc_options: []
|
176
|
+
require_paths:
|
177
|
+
- lib
|
178
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
179
|
+
none: false
|
180
|
+
requirements:
|
181
|
+
- - ! '>='
|
182
|
+
- !ruby/object:Gem::Version
|
183
|
+
version: '0'
|
184
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
185
|
+
none: false
|
186
|
+
requirements:
|
187
|
+
- - ! '>='
|
188
|
+
- !ruby/object:Gem::Version
|
189
|
+
version: '0'
|
190
|
+
requirements: []
|
191
|
+
rubyforge_project:
|
192
|
+
rubygems_version: 1.8.24
|
193
|
+
signing_key:
|
194
|
+
specification_version: 3
|
195
|
+
summary: Upsert for MySQL, PostgreSQL, and SQLite. Finally, all those SQL MERGE tricks
|
196
|
+
codified.
|
197
|
+
test_files:
|
198
|
+
- test/helper.rb
|
199
|
+
- test/shared_examples.rb
|
200
|
+
- test/test_mysql2.rb
|
201
|
+
- test/test_pg.rb
|
202
|
+
- test/test_sqlite.rb
|
203
|
+
- test/test_upsert.rb
|
204
|
+
has_rdoc:
|