upsert 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +17 -0
- data/Gemfile +4 -0
- data/LICENSE +22 -0
- data/README.md +70 -0
- data/Rakefile +25 -0
- data/lib/upsert.rb +41 -0
- data/lib/upsert/buffer.rb +58 -0
- data/lib/upsert/buffer/mysql2_client.rb +47 -0
- data/lib/upsert/buffer/pg_connection.rb +94 -0
- data/lib/upsert/buffer/pg_connection/column_definition.rb +34 -0
- data/lib/upsert/buffer/sqlite3_database.rb +44 -0
- data/lib/upsert/quoter.rb +15 -0
- data/lib/upsert/row.rb +41 -0
- data/lib/upsert/version.rb +3 -0
- data/test/helper.rb +54 -0
- data/test/shared_examples.rb +94 -0
- data/test/test_mysql2.rb +28 -0
- data/test/test_pg.rb +29 -0
- data/test/test_sqlite.rb +24 -0
- data/test/test_upsert.rb +7 -0
- data/upsert.gemspec +26 -0
- metadata +204 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Seamus Abshere
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
# Upsert
|
2
|
+
|
3
|
+
Finally, all those SQL MERGE tricks codified.
|
4
|
+
|
5
|
+
## Supported databases
|
6
|
+
|
7
|
+
### MySQL
|
8
|
+
|
9
|
+
# http://dev.mysql.com/doc/refman/5.0/en/insert-on-duplicate.html
|
10
|
+
INSERT INTO table (a,b,c) VALUES (1,2,3)
|
11
|
+
ON DUPLICATE KEY UPDATE c=c+1;
|
12
|
+
|
13
|
+
### PostgreSQL
|
14
|
+
|
15
|
+
#### Used
|
16
|
+
|
17
|
+
# http://www.postgresql.org/docs/current/interactive/plpgsql-control-structures.html#PLPGSQL-ERROR-TRAPPING
|
18
|
+
CREATE TABLE db (a INT PRIMARY KEY, b TEXT);
|
19
|
+
CREATE FUNCTION merge_db(key INT, data TEXT) RETURNS VOID AS
|
20
|
+
$$
|
21
|
+
BEGIN
|
22
|
+
LOOP
|
23
|
+
-- first try to update the key
|
24
|
+
UPDATE db SET b = data WHERE a = key;
|
25
|
+
IF found THEN
|
26
|
+
RETURN;
|
27
|
+
END IF;
|
28
|
+
-- not there, so try to insert the key
|
29
|
+
-- if someone else inserts the same key concurrently,
|
30
|
+
-- we could get a unique-key failure
|
31
|
+
BEGIN
|
32
|
+
INSERT INTO db(a,b) VALUES (key, data);
|
33
|
+
RETURN;
|
34
|
+
EXCEPTION WHEN unique_violation THEN
|
35
|
+
-- Do nothing, and loop to try the UPDATE again.
|
36
|
+
END;
|
37
|
+
END LOOP;
|
38
|
+
END;
|
39
|
+
$$
|
40
|
+
LANGUAGE plpgsql;
|
41
|
+
SELECT merge_db(1, 'david');
|
42
|
+
SELECT merge_db(1, 'dennis');
|
43
|
+
|
44
|
+
#### Alternatives (not used)
|
45
|
+
|
46
|
+
# http://stackoverflow.com/questions/1109061/insert-on-duplicate-update-postgresql
|
47
|
+
UPDATE table SET field='C', field2='Z' WHERE id=3;
|
48
|
+
INSERT INTO table (id, field, field2)
|
49
|
+
SELECT 3, 'C', 'Z'
|
50
|
+
WHERE NOT EXISTS (SELECT 1 FROM table WHERE id=3);
|
51
|
+
|
52
|
+
# http://stackoverflow.com/questions/5269590/why-doesnt-this-rule-prevent-duplicate-key-violations
|
53
|
+
BEGIN;
|
54
|
+
CREATE TEMP TABLE stage_data(key_column, data_columns...) ON COMMIT DROP;
|
55
|
+
\copy stage_data from data.csv with csv header
|
56
|
+
-- prevent any other updates while we are merging input (omit this if you don't need it)
|
57
|
+
LOCK target_data IN SHARE ROW EXCLUSIVE MODE;
|
58
|
+
-- insert into target table
|
59
|
+
INSERT INTO target_data(key_column, data_columns...)
|
60
|
+
SELECT key_column, data_columns...
|
61
|
+
FROM stage_data
|
62
|
+
WHERE NOT EXISTS (SELECT 1 FROM target_data
|
63
|
+
WHERE target_data.key_column = stage_data.key_column)
|
64
|
+
END;
|
65
|
+
|
66
|
+
### Sqlite
|
67
|
+
|
68
|
+
# http://stackoverflow.com/questions/2717590/sqlite-upsert-on-duplicate-key-update
|
69
|
+
INSERT OR IGNORE INTO visits VALUES ($ip, 0);
|
70
|
+
UPDATE visits SET hits = hits + 1 WHERE ip LIKE $ip;
|
data/Rakefile
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
|
4
|
+
require 'rake'
|
5
|
+
require 'rake/testtask'
|
6
|
+
Rake::TestTask.new(:_test) do |test|
|
7
|
+
test.libs << 'lib' << 'test'
|
8
|
+
test.pattern = 'test/**/test_*.rb'
|
9
|
+
test.verbose = true
|
10
|
+
end
|
11
|
+
|
12
|
+
task :test_each_db_adapter do
|
13
|
+
%w{ mysql2 sqlite pg }.each do |database|
|
14
|
+
puts
|
15
|
+
puts "#{'*'*10} Running #{database} tests"
|
16
|
+
puts
|
17
|
+
puts `rake _test TEST=test/test_#{database}.rb`
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
task :default => :test_each_db_adapter
|
22
|
+
task :test => :test_each_db_adapter
|
23
|
+
|
24
|
+
require 'yard'
|
25
|
+
YARD::Rake::YardocTask.new
|
data/lib/upsert.rb
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'upsert/version'
|
2
|
+
require 'upsert/buffer'
|
3
|
+
require 'upsert/quoter'
|
4
|
+
require 'upsert/row'
|
5
|
+
require 'upsert/buffer/mysql2_client'
|
6
|
+
require 'upsert/buffer/pg_connection'
|
7
|
+
require 'upsert/buffer/sqlite3_database'
|
8
|
+
|
9
|
+
class Upsert
|
10
|
+
INFINITY = 1.0/0
|
11
|
+
SINGLE_QUOTE = %{'}
|
12
|
+
DOUBLE_QUOTE = %{"}
|
13
|
+
BACKTICK = %{`}
|
14
|
+
|
15
|
+
attr_reader :buffer
|
16
|
+
|
17
|
+
def initialize(connection, table_name)
|
18
|
+
@multi_mutex = Mutex.new
|
19
|
+
@buffer = Buffer.for connection, table_name
|
20
|
+
end
|
21
|
+
|
22
|
+
def row(selector, document)
|
23
|
+
buffer.add selector, document
|
24
|
+
end
|
25
|
+
|
26
|
+
def cleanup
|
27
|
+
buffer.cleanup
|
28
|
+
end
|
29
|
+
|
30
|
+
def multi(&blk)
|
31
|
+
@multi_mutex.synchronize do
|
32
|
+
begin
|
33
|
+
buffer.async = true
|
34
|
+
instance_eval(&blk)
|
35
|
+
buffer.cleanup
|
36
|
+
ensure
|
37
|
+
buffer.async = nil
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
class Upsert
|
2
|
+
class Buffer
|
3
|
+
class << self
|
4
|
+
def for(connection, table_name)
|
5
|
+
const_get(connection.class.name.gsub(/\W+/, '_')).new connection, table_name
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
attr_reader :connection
|
10
|
+
attr_reader :table_name
|
11
|
+
attr_reader :rows
|
12
|
+
attr_writer :async
|
13
|
+
|
14
|
+
def initialize(connection, table_name)
|
15
|
+
@connection = connection
|
16
|
+
@table_name = table_name
|
17
|
+
@rows = []
|
18
|
+
end
|
19
|
+
|
20
|
+
def async?
|
21
|
+
!!@async
|
22
|
+
end
|
23
|
+
|
24
|
+
def add(selector, document)
|
25
|
+
rows << Row.new(selector, document)
|
26
|
+
if sql = chunk
|
27
|
+
execute sql
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def clear
|
32
|
+
while sql = chunk
|
33
|
+
execute sql
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def chunk
|
38
|
+
return if rows.empty?
|
39
|
+
targets = []
|
40
|
+
sql = nil
|
41
|
+
begin
|
42
|
+
targets << rows.pop
|
43
|
+
last_sql = sql
|
44
|
+
sql = compose(targets)
|
45
|
+
end until rows.empty? or targets.length >= max_targets or sql.length > max_length
|
46
|
+
if sql.length > max_length
|
47
|
+
raise if last_sql.nil?
|
48
|
+
sql = last_sql
|
49
|
+
rows << targets.pop
|
50
|
+
end
|
51
|
+
sql
|
52
|
+
end
|
53
|
+
|
54
|
+
def cleanup
|
55
|
+
clear
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
class Upsert
|
2
|
+
class Buffer
|
3
|
+
class Mysql2_Client < Buffer
|
4
|
+
def compose(targets)
|
5
|
+
columns = targets.first.columns
|
6
|
+
row_inserts = targets.map { |row| row.inserts }
|
7
|
+
column_tautologies = columns.map do |k|
|
8
|
+
[ quote_ident(k), "VALUES(#{quote_ident(k)})" ].join('=')
|
9
|
+
end
|
10
|
+
sql = <<-EOS
|
11
|
+
INSERT INTO "#{table_name}" (#{quote_idents(columns)}) VALUES (#{row_inserts.map { |row_insert| quote_values(row_insert) }.join('),(') })
|
12
|
+
ON DUPLICATE KEY UPDATE #{column_tautologies.join(',')};
|
13
|
+
EOS
|
14
|
+
sql
|
15
|
+
end
|
16
|
+
|
17
|
+
def execute(sql)
|
18
|
+
connection.query sql
|
19
|
+
end
|
20
|
+
|
21
|
+
def max_targets
|
22
|
+
INFINITY
|
23
|
+
end
|
24
|
+
|
25
|
+
def max_length
|
26
|
+
@max_length ||= connection.query("SHOW VARIABLES LIKE 'max_allowed_packet'", :as => :hash).first['Value'].to_i
|
27
|
+
end
|
28
|
+
|
29
|
+
include Quoter
|
30
|
+
|
31
|
+
def quote_value(v)
|
32
|
+
case v
|
33
|
+
when NilClass
|
34
|
+
'NULL'
|
35
|
+
when String, Symbol
|
36
|
+
SINGLE_QUOTE + connection.escape(v.to_s) + SINGLE_QUOTE
|
37
|
+
else
|
38
|
+
v
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def quote_ident(k)
|
43
|
+
BACKTICK + connection.escape(k.to_s) + BACKTICK
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
require 'upsert/buffer/pg_connection/column_definition'
|
2
|
+
|
3
|
+
class Upsert
|
4
|
+
class Buffer
|
5
|
+
class PG_Connection < Buffer
|
6
|
+
attr_reader :db_function_name
|
7
|
+
|
8
|
+
def compose(targets)
|
9
|
+
target = targets.first
|
10
|
+
unless created_db_function?
|
11
|
+
create_db_function target
|
12
|
+
end
|
13
|
+
hsh = target.to_hash
|
14
|
+
ordered_args = column_definitions.map do |c|
|
15
|
+
if hsh.has_key? c.name
|
16
|
+
hsh[c.name]
|
17
|
+
else
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
end
|
21
|
+
%{ SELECT #{db_function_name}(#{quote_values(ordered_args)}) }
|
22
|
+
end
|
23
|
+
|
24
|
+
def execute(sql)
|
25
|
+
connection.exec sql
|
26
|
+
end
|
27
|
+
|
28
|
+
def max_length
|
29
|
+
INFINITY
|
30
|
+
end
|
31
|
+
|
32
|
+
def max_targets
|
33
|
+
1
|
34
|
+
end
|
35
|
+
|
36
|
+
include Quoter
|
37
|
+
|
38
|
+
def quote_ident(k)
|
39
|
+
SINGLE_QUOTE + connection.quote_ident(k) + SINGLE_QUOTE
|
40
|
+
end
|
41
|
+
|
42
|
+
# FIXME escape_bytea with (v, k = nil)
|
43
|
+
def quote_value(v)
|
44
|
+
case v
|
45
|
+
when NilClass
|
46
|
+
'NULL'
|
47
|
+
when String, Symbol
|
48
|
+
SINGLE_QUOTE + connection.escape_string(v.to_s) + SINGLE_QUOTE
|
49
|
+
else
|
50
|
+
v
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def column_definitions
|
55
|
+
@column_definitions ||= ColumnDefinition.all(connection, table_name)
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
|
60
|
+
def created_db_function?
|
61
|
+
!!@created_db_function_query
|
62
|
+
end
|
63
|
+
|
64
|
+
def create_db_function(example_row)
|
65
|
+
@db_function_name = "pg_temp.merge_#{table_name}_#{Kernel.rand(1e11)}"
|
66
|
+
execute <<-EOS
|
67
|
+
CREATE FUNCTION #{db_function_name}(#{column_definitions.map { |c| "#{c.name}_input #{c.sql_type} DEFAULT #{c.default || 'NULL'}" }.join(',') }) RETURNS VOID AS
|
68
|
+
$$
|
69
|
+
BEGIN
|
70
|
+
LOOP
|
71
|
+
-- first try to update the key
|
72
|
+
UPDATE #{table_name} SET #{column_definitions.map { |c| "#{c.name} = #{c.name}_input" }.join(',')} WHERE #{example_row.selector.keys.map { |k| "#{k} = #{k}_input" }.join(' AND ') };
|
73
|
+
IF found THEN
|
74
|
+
RETURN;
|
75
|
+
END IF;
|
76
|
+
-- not there, so try to insert the key
|
77
|
+
-- if someone else inserts the same key concurrently,
|
78
|
+
-- we could get a unique-key failure
|
79
|
+
BEGIN
|
80
|
+
INSERT INTO #{table_name}(#{column_definitions.map { |c| c.name }.join(',')}) VALUES (#{column_definitions.map { |c| "#{c.name}_input" }.join(',')});
|
81
|
+
RETURN;
|
82
|
+
EXCEPTION WHEN unique_violation THEN
|
83
|
+
-- Do nothing, and loop to try the UPDATE again.
|
84
|
+
END;
|
85
|
+
END LOOP;
|
86
|
+
END;
|
87
|
+
$$
|
88
|
+
LANGUAGE plpgsql;
|
89
|
+
EOS
|
90
|
+
@created_db_function_query = true
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
class Upsert
|
2
|
+
class Buffer
|
3
|
+
class PG_Connection < Buffer
|
4
|
+
class ColumnDefinition
|
5
|
+
class << self
|
6
|
+
def all(connection, table_name)
|
7
|
+
# activerecord-3.2.5/lib/active_record/connection_adapters/postgresql_adapter.rb#column_definitions
|
8
|
+
res = connection.exec <<-EOS
|
9
|
+
SELECT a.attname AS name, format_type(a.atttypid, a.atttypmod) AS sql_type, d.adsrc AS default
|
10
|
+
FROM pg_attribute a LEFT JOIN pg_attrdef d
|
11
|
+
ON a.attrelid = d.adrelid AND a.attnum = d.adnum
|
12
|
+
WHERE a.attrelid = '#{connection.quote_ident(table_name.to_s)}'::regclass
|
13
|
+
AND a.attnum > 0 AND NOT a.attisdropped
|
14
|
+
ORDER BY a.attnum
|
15
|
+
EOS
|
16
|
+
res.map do |row|
|
17
|
+
new row['name'], row['sql_type'], row['default']
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
attr_reader :name
|
23
|
+
attr_reader :sql_type
|
24
|
+
attr_reader :default
|
25
|
+
|
26
|
+
def initialize(name, sql_type, default)
|
27
|
+
@name = name
|
28
|
+
@sql_type = sql_type
|
29
|
+
@default = default
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
class Upsert
|
2
|
+
class Buffer
|
3
|
+
class SQLite3_Database < Buffer
|
4
|
+
def compose(targets)
|
5
|
+
target = targets.first
|
6
|
+
parts = []
|
7
|
+
parts << %{ INSERT OR IGNORE INTO "#{table_name}" (#{quote_idents(target.columns)}) VALUES (#{quote_values(target.inserts)}) }
|
8
|
+
if target.updates.length > 0
|
9
|
+
parts << %{ UPDATE "#{table_name}" SET #{quote_pairs(target.updates)} WHERE #{quote_pairs(target.selector)} }
|
10
|
+
end
|
11
|
+
parts.join(';')
|
12
|
+
end
|
13
|
+
|
14
|
+
def execute(sql)
|
15
|
+
connection.execute_batch sql
|
16
|
+
end
|
17
|
+
|
18
|
+
def max_targets
|
19
|
+
1
|
20
|
+
end
|
21
|
+
|
22
|
+
def max_length
|
23
|
+
INFINITY
|
24
|
+
end
|
25
|
+
|
26
|
+
include Quoter
|
27
|
+
|
28
|
+
def quote_value(v)
|
29
|
+
case v
|
30
|
+
when NilClass
|
31
|
+
'NULL'
|
32
|
+
when String, Symbol
|
33
|
+
SINGLE_QUOTE + SQLite3::Database.quote(v.to_s) + SINGLE_QUOTE
|
34
|
+
else
|
35
|
+
v
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def quote_ident(k)
|
40
|
+
DOUBLE_QUOTE + SQLite3::Database.quote(k.to_s) + DOUBLE_QUOTE
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class Upsert
|
2
|
+
module Quoter
|
3
|
+
def quote_idents(idents)
|
4
|
+
idents.map { |k| quote_ident(k) }.join(',')
|
5
|
+
end
|
6
|
+
|
7
|
+
def quote_values(values)
|
8
|
+
values.map { |v| quote_value(v) }.join(',')
|
9
|
+
end
|
10
|
+
|
11
|
+
def quote_pairs(pairs)
|
12
|
+
pairs.map { |k, v| [quote_ident(k),quote_value(v)].join('=') }.join(',')
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
data/lib/upsert/row.rb
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
class Upsert
|
2
|
+
class Row
|
3
|
+
attr_reader :selector
|
4
|
+
attr_reader :document
|
5
|
+
|
6
|
+
def initialize(selector, document)
|
7
|
+
@selector = selector
|
8
|
+
@document = document
|
9
|
+
end
|
10
|
+
|
11
|
+
def columns
|
12
|
+
@columns ||= (selector.keys+document.keys).uniq
|
13
|
+
end
|
14
|
+
|
15
|
+
def pairs
|
16
|
+
@pairs ||= columns.map do |k|
|
17
|
+
value = if selector.has_key?(k)
|
18
|
+
selector[k]
|
19
|
+
else
|
20
|
+
document[k]
|
21
|
+
end
|
22
|
+
[ k, value ]
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def inserts
|
27
|
+
@inserts ||= pairs.map { |_, v| v }
|
28
|
+
end
|
29
|
+
|
30
|
+
def updates
|
31
|
+
@updates ||= pairs.reject { |k, _| selector.has_key?(k) }
|
32
|
+
end
|
33
|
+
|
34
|
+
def to_hash
|
35
|
+
@to_hash ||= pairs.inject({}) do |memo, (k, v)|
|
36
|
+
memo[k.to_s] = v
|
37
|
+
memo
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler/setup'
|
3
|
+
require 'minitest/spec'
|
4
|
+
require 'minitest/autorun'
|
5
|
+
require 'minitest/reporters'
|
6
|
+
MiniTest::Unit.runner = MiniTest::SuiteRunner.new
|
7
|
+
MiniTest::Unit.runner.reporters << MiniTest::Reporters::SpecReporter.new
|
8
|
+
|
9
|
+
require 'active_record'
|
10
|
+
require 'active_record_inline_schema'
|
11
|
+
|
12
|
+
# require 'logger'
|
13
|
+
# ActiveRecord::Base.logger = Logger.new($stdout)
|
14
|
+
# ActiveRecord::Base.logger.level = Logger::DEBUG
|
15
|
+
|
16
|
+
class Pet < ActiveRecord::Base
|
17
|
+
self.primary_key = 'name'
|
18
|
+
col :name
|
19
|
+
col :gender
|
20
|
+
end
|
21
|
+
|
22
|
+
require 'upsert'
|
23
|
+
|
24
|
+
MiniTest::Spec.class_eval do
|
25
|
+
def self.shared_examples
|
26
|
+
@shared_examples ||= {}
|
27
|
+
end
|
28
|
+
|
29
|
+
def assert_creates(model, expected_records)
|
30
|
+
expected_records.each do |conditions|
|
31
|
+
model.where(conditions).count.must_equal 0
|
32
|
+
end
|
33
|
+
yield
|
34
|
+
expected_records.each do |conditions|
|
35
|
+
model.where(conditions).count.must_equal 1
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
module MiniTest::Spec::SharedExamples
|
42
|
+
def shared_examples_for(desc, &block)
|
43
|
+
MiniTest::Spec.shared_examples[desc] = block
|
44
|
+
end
|
45
|
+
|
46
|
+
def it_behaves_like(desc)
|
47
|
+
self.instance_eval do
|
48
|
+
MiniTest::Spec.shared_examples[desc].call
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
Object.class_eval { include(MiniTest::Spec::SharedExamples) }
|
54
|
+
require 'shared_examples'
|
@@ -0,0 +1,94 @@
|
|
1
|
+
shared_examples_for :database do
|
2
|
+
describe :row do
|
3
|
+
it "works for a single row (base case)" do
|
4
|
+
upsert = Upsert.new connection, :pets
|
5
|
+
assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
|
6
|
+
upsert.row({:name => 'Jerry'}, {:gender => 'male'})
|
7
|
+
end
|
8
|
+
end
|
9
|
+
it "works for a single row (not changing anything)" do
|
10
|
+
upsert = Upsert.new connection, :pets
|
11
|
+
assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
|
12
|
+
upsert.row({:name => 'Jerry'}, {:gender => 'male'})
|
13
|
+
upsert.row({:name => 'Jerry'}, {:gender => 'male'})
|
14
|
+
end
|
15
|
+
end
|
16
|
+
it "works for a single row (changing something)" do
|
17
|
+
upsert = Upsert.new connection, :pets
|
18
|
+
assert_creates(Pet, [{:name => 'Jerry', :gender => 'neutered'}]) do
|
19
|
+
upsert.row({:name => 'Jerry'}, {:gender => 'male'})
|
20
|
+
upsert.row({:name => 'Jerry'}, {:gender => 'neutered'})
|
21
|
+
end
|
22
|
+
Pet.where(:gender => 'male').count.must_equal 0
|
23
|
+
end
|
24
|
+
|
25
|
+
it "works for a single row with implicit nulls" do
|
26
|
+
upsert = Upsert.new connection, :pets
|
27
|
+
assert_creates(Pet, [{:name => 'Inky', :gender => nil}]) do
|
28
|
+
upsert.row({:name => 'Inky'}, {})
|
29
|
+
upsert.row({:name => 'Inky'}, {})
|
30
|
+
end
|
31
|
+
end
|
32
|
+
it "works for a single row with explicit nulls" do
|
33
|
+
upsert = Upsert.new connection, :pets
|
34
|
+
assert_creates(Pet, [{:name => 'Inky', :gender => nil}]) do
|
35
|
+
upsert.row({:name => 'Inky'}, {:gender => nil})
|
36
|
+
upsert.row({:name => 'Inky'}, {:gender => nil})
|
37
|
+
end
|
38
|
+
end
|
39
|
+
# it "works for a single row upserted many times" do
|
40
|
+
# assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
|
41
|
+
# ts = (0..5).map do
|
42
|
+
# Thread.new do
|
43
|
+
# upsert = Upsert.new new_connection, :pets
|
44
|
+
# upsert.row({:name => 'Jerry'}, {:gender => 'male'})
|
45
|
+
# end
|
46
|
+
# end
|
47
|
+
# ts.each { |t| t.join }
|
48
|
+
# end
|
49
|
+
# end
|
50
|
+
end
|
51
|
+
describe :multi do
|
52
|
+
it "works for multiple rows (base case)" do
|
53
|
+
upsert = Upsert.new connection, :pets
|
54
|
+
assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
|
55
|
+
upsert.multi do
|
56
|
+
row({:name => 'Jerry'}, :gender => 'male')
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
it "works for multiple rows (not changing anything)" do
|
61
|
+
upsert = Upsert.new connection, :pets
|
62
|
+
assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
|
63
|
+
upsert.multi do
|
64
|
+
row({:name => 'Jerry'}, :gender => 'male')
|
65
|
+
row({:name => 'Jerry'}, :gender => 'male')
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
it "works for multiple rows (changing something)" do
|
70
|
+
upsert = Upsert.new connection, :pets
|
71
|
+
assert_creates(Pet, [{:name => 'Jerry', :gender => 'neutered'}]) do
|
72
|
+
upsert.multi do
|
73
|
+
row({:name => 'Jerry'}, :gender => 'male')
|
74
|
+
row({:name => 'Jerry'}, :gender => 'neutered')
|
75
|
+
end
|
76
|
+
end
|
77
|
+
Pet.where(:gender => 'male').count.must_equal 0
|
78
|
+
end
|
79
|
+
# it "works for multiple rows upserted many times" do
|
80
|
+
# assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
|
81
|
+
# ts = (0..5).map do
|
82
|
+
# Thread.new do
|
83
|
+
# upsert = Upsert.new new_connection, :pets
|
84
|
+
# upsert.multi do
|
85
|
+
# row({:name => 'Jerry'}, :gender => 'male')
|
86
|
+
# row({:name => 'Jerry'}, :gender => 'male')
|
87
|
+
# end
|
88
|
+
# end
|
89
|
+
# end
|
90
|
+
# ts.each { |t| t.join }
|
91
|
+
# end
|
92
|
+
# end
|
93
|
+
end
|
94
|
+
end
|
data/test/test_mysql2.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'mysql2'
|
3
|
+
|
4
|
+
system %{ mysql -u root -ppassword -e "DROP DATABASE IF EXISTS test_upsert; CREATE DATABASE test_upsert CHARSET utf8" }
|
5
|
+
ActiveRecord::Base.establish_connection :adapter => 'mysql2', :username => 'root', :password => 'password', :database => 'test_upsert'
|
6
|
+
|
7
|
+
describe "upserting on mysql2" do
|
8
|
+
before do
|
9
|
+
ActiveRecord::Base.connection.drop_table Pet.table_name rescue nil
|
10
|
+
Pet.auto_upgrade!
|
11
|
+
@opened_connections = []
|
12
|
+
@connection = new_connection
|
13
|
+
end
|
14
|
+
after do
|
15
|
+
@opened_connections.each { |c| c.close }
|
16
|
+
end
|
17
|
+
def new_connection
|
18
|
+
c = Mysql2::Client.new(:username => 'root', :password => 'password', :database => 'test_upsert')
|
19
|
+
@opened_connections << c
|
20
|
+
c
|
21
|
+
end
|
22
|
+
def connection
|
23
|
+
@connection
|
24
|
+
end
|
25
|
+
|
26
|
+
it_behaves_like :database
|
27
|
+
|
28
|
+
end
|
data/test/test_pg.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'pg'
|
3
|
+
|
4
|
+
system %{ dropdb test_upsert }
|
5
|
+
system %{ createdb test_upsert }
|
6
|
+
ActiveRecord::Base.establish_connection :adapter => 'postgresql', :database => 'test_upsert'
|
7
|
+
|
8
|
+
describe "upserting on postgresql" do
|
9
|
+
before do
|
10
|
+
ActiveRecord::Base.connection.drop_table Pet.table_name rescue nil
|
11
|
+
Pet.auto_upgrade!
|
12
|
+
@opened_connections = []
|
13
|
+
@connection = new_connection
|
14
|
+
end
|
15
|
+
after do
|
16
|
+
@opened_connections.each { |c| c.finish }
|
17
|
+
end
|
18
|
+
def new_connection
|
19
|
+
c = PG.connect(:dbname => 'test_upsert')
|
20
|
+
@opened_connections << c
|
21
|
+
c
|
22
|
+
end
|
23
|
+
def connection
|
24
|
+
@connection
|
25
|
+
end
|
26
|
+
|
27
|
+
it_behaves_like :database
|
28
|
+
|
29
|
+
end
|
data/test/test_sqlite.rb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'sqlite3'
|
3
|
+
|
4
|
+
db_path = File.expand_path('../../tmp/test.sqlite3', __FILE__)
|
5
|
+
FileUtils.mkdir_p File.dirname(db_path)
|
6
|
+
FileUtils.rm_f db_path
|
7
|
+
ActiveRecord::Base.establish_connection :adapter => 'sqlite3', :database => db_path
|
8
|
+
|
9
|
+
describe "upserting on sqlite" do
|
10
|
+
before do
|
11
|
+
ActiveRecord::Base.connection.drop_table Pet.table_name rescue nil
|
12
|
+
Pet.auto_upgrade!
|
13
|
+
@connection = new_connection
|
14
|
+
end
|
15
|
+
def new_connection
|
16
|
+
db_path = File.expand_path('../../tmp/test.sqlite3', __FILE__)
|
17
|
+
SQLite3::Database.open(db_path)
|
18
|
+
end
|
19
|
+
def connection
|
20
|
+
@connection
|
21
|
+
end
|
22
|
+
|
23
|
+
it_behaves_like :database
|
24
|
+
end
|
data/test/test_upsert.rb
ADDED
data/upsert.gemspec
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/upsert/version', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.authors = ["Seamus Abshere"]
|
6
|
+
gem.email = ["seamus@abshere.net"]
|
7
|
+
gem.description = %q{Upsert for MySQL, PostgreSQL, and SQLite. Codifies various SQL MERGE tricks like MySQL's ON DUPLICATE KEY UPDATE, PostgreSQL's CREATE FUNCTION merge_db, and SQLite's INSERT OR IGNORE.}
|
8
|
+
gem.summary = %q{Upsert for MySQL, PostgreSQL, and SQLite. Finally, all those SQL MERGE tricks codified.}
|
9
|
+
gem.homepage = "https://github.com/seamusabshere/upsert"
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
|
+
gem.name = "upsert"
|
15
|
+
gem.require_paths = ["lib"]
|
16
|
+
gem.version = Upsert::VERSION
|
17
|
+
|
18
|
+
gem.add_development_dependency 'sqlite3'
|
19
|
+
gem.add_development_dependency 'mysql2'
|
20
|
+
gem.add_development_dependency 'pg'
|
21
|
+
gem.add_development_dependency 'activerecord' # testing only
|
22
|
+
gem.add_development_dependency 'active_record_inline_schema'
|
23
|
+
gem.add_development_dependency 'minitest'
|
24
|
+
gem.add_development_dependency 'minitest-reporters'
|
25
|
+
gem.add_development_dependency 'yard'
|
26
|
+
end
|
metadata
ADDED
@@ -0,0 +1,204 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: upsert
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Seamus Abshere
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-06-13 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: sqlite3
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: mysql2
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: pg
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: activerecord
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: active_record_inline_schema
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
type: :development
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: minitest
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
102
|
+
type: :development
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
- !ruby/object:Gem::Dependency
|
111
|
+
name: minitest-reporters
|
112
|
+
requirement: !ruby/object:Gem::Requirement
|
113
|
+
none: false
|
114
|
+
requirements:
|
115
|
+
- - ! '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
none: false
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: yard
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '0'
|
134
|
+
type: :development
|
135
|
+
prerelease: false
|
136
|
+
version_requirements: !ruby/object:Gem::Requirement
|
137
|
+
none: false
|
138
|
+
requirements:
|
139
|
+
- - ! '>='
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '0'
|
142
|
+
description: Upsert for MySQL, PostgreSQL, and SQLite. Codifies various SQL MERGE
|
143
|
+
tricks like MySQL's ON DUPLICATE KEY UPDATE, PostgreSQL's CREATE FUNCTION merge_db,
|
144
|
+
and SQLite's INSERT OR IGNORE.
|
145
|
+
email:
|
146
|
+
- seamus@abshere.net
|
147
|
+
executables: []
|
148
|
+
extensions: []
|
149
|
+
extra_rdoc_files: []
|
150
|
+
files:
|
151
|
+
- .gitignore
|
152
|
+
- Gemfile
|
153
|
+
- LICENSE
|
154
|
+
- README.md
|
155
|
+
- Rakefile
|
156
|
+
- lib/upsert.rb
|
157
|
+
- lib/upsert/buffer.rb
|
158
|
+
- lib/upsert/buffer/mysql2_client.rb
|
159
|
+
- lib/upsert/buffer/pg_connection.rb
|
160
|
+
- lib/upsert/buffer/pg_connection/column_definition.rb
|
161
|
+
- lib/upsert/buffer/sqlite3_database.rb
|
162
|
+
- lib/upsert/quoter.rb
|
163
|
+
- lib/upsert/row.rb
|
164
|
+
- lib/upsert/version.rb
|
165
|
+
- test/helper.rb
|
166
|
+
- test/shared_examples.rb
|
167
|
+
- test/test_mysql2.rb
|
168
|
+
- test/test_pg.rb
|
169
|
+
- test/test_sqlite.rb
|
170
|
+
- test/test_upsert.rb
|
171
|
+
- upsert.gemspec
|
172
|
+
homepage: https://github.com/seamusabshere/upsert
|
173
|
+
licenses: []
|
174
|
+
post_install_message:
|
175
|
+
rdoc_options: []
|
176
|
+
require_paths:
|
177
|
+
- lib
|
178
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
179
|
+
none: false
|
180
|
+
requirements:
|
181
|
+
- - ! '>='
|
182
|
+
- !ruby/object:Gem::Version
|
183
|
+
version: '0'
|
184
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
185
|
+
none: false
|
186
|
+
requirements:
|
187
|
+
- - ! '>='
|
188
|
+
- !ruby/object:Gem::Version
|
189
|
+
version: '0'
|
190
|
+
requirements: []
|
191
|
+
rubyforge_project:
|
192
|
+
rubygems_version: 1.8.24
|
193
|
+
signing_key:
|
194
|
+
specification_version: 3
|
195
|
+
summary: Upsert for MySQL, PostgreSQL, and SQLite. Finally, all those SQL MERGE tricks
|
196
|
+
codified.
|
197
|
+
test_files:
|
198
|
+
- test/helper.rb
|
199
|
+
- test/shared_examples.rb
|
200
|
+
- test/test_mysql2.rb
|
201
|
+
- test/test_pg.rb
|
202
|
+
- test/test_sqlite.rb
|
203
|
+
- test/test_upsert.rb
|
204
|
+
has_rdoc:
|