upsert 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.yardopts +2 -0
- data/README.md +54 -11
- data/Rakefile +1 -1
- data/lib/upsert.rb +39 -16
- data/lib/upsert/binary.rb +7 -0
- data/lib/upsert/buffer.rb +14 -22
- data/lib/upsert/buffer/mysql2_client.rb +103 -20
- data/lib/upsert/buffer/pg_connection.rb +33 -40
- data/lib/upsert/buffer/pg_connection/column_definition.rb +28 -2
- data/lib/upsert/buffer/sqlite3_database.rb +25 -23
- data/lib/upsert/quoter.rb +29 -1
- data/lib/upsert/row.rb +29 -13
- data/lib/upsert/version.rb +1 -1
- data/test/helper.rb +80 -3
- data/test/shared/binary.rb +20 -0
- data/test/shared/correctness.rb +48 -0
- data/test/{shared_examples.rb → shared/database.rb} +9 -9
- data/test/shared/multibyte.rb +26 -0
- data/test/shared/speed.rb +72 -0
- data/test/shared/timezones.rb +27 -0
- data/test/test_active_record_connection_adapter.rb +36 -0
- data/test/test_mysql2.rb +11 -2
- data/test/test_pg.rb +11 -2
- data/test/test_sqlite.rb +20 -4
- data/upsert.gemspec +2 -0
- metadata +50 -6
- data/test/test_upsert.rb +0 -7
@@ -1,10 +1,34 @@
|
|
1
1
|
class Upsert
|
2
2
|
class Buffer
|
3
3
|
class PG_Connection < Buffer
|
4
|
+
# @private
|
5
|
+
# activerecord-3.2.5/lib/active_record/connection_adapters/postgresql_adapter.rb#column_definitions
|
4
6
|
class ColumnDefinition
|
5
7
|
class << self
|
8
|
+
def auto_increment_primary_key(connection, table_name)
|
9
|
+
res = connection.exec <<-EOS
|
10
|
+
SELECT attr.attname, seq.relname
|
11
|
+
FROM pg_class seq,
|
12
|
+
pg_attribute attr,
|
13
|
+
pg_depend dep,
|
14
|
+
pg_namespace name,
|
15
|
+
pg_constraint cons
|
16
|
+
WHERE seq.oid = dep.objid
|
17
|
+
AND seq.relkind = 'S'
|
18
|
+
AND attr.attrelid = dep.refobjid
|
19
|
+
AND attr.attnum = dep.refobjsubid
|
20
|
+
AND attr.attrelid = cons.conrelid
|
21
|
+
AND attr.attnum = cons.conkey[1]
|
22
|
+
AND cons.contype = 'p'
|
23
|
+
AND dep.refobjid = '#{connection.quote_ident(table_name.to_s)}'::regclass
|
24
|
+
EOS
|
25
|
+
if hit = res.first
|
26
|
+
hit['attname']
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
6
30
|
def all(connection, table_name)
|
7
|
-
|
31
|
+
auto_increment_primary_key = auto_increment_primary_key(connection, table_name)
|
8
32
|
res = connection.exec <<-EOS
|
9
33
|
SELECT a.attname AS name, format_type(a.atttypid, a.atttypmod) AS sql_type, d.adsrc AS default
|
10
34
|
FROM pg_attribute a LEFT JOIN pg_attrdef d
|
@@ -13,7 +37,9 @@ SELECT a.attname AS name, format_type(a.atttypid, a.atttypmod) AS sql_type, d.ad
|
|
13
37
|
AND a.attnum > 0 AND NOT a.attisdropped
|
14
38
|
ORDER BY a.attnum
|
15
39
|
EOS
|
16
|
-
res.
|
40
|
+
res.reject do |row|
|
41
|
+
row['name'] == auto_increment_primary_key
|
42
|
+
end.map do |row|
|
17
43
|
new row['name'], row['sql_type'], row['default']
|
18
44
|
end
|
19
45
|
end
|
@@ -1,44 +1,46 @@
|
|
1
1
|
class Upsert
|
2
2
|
class Buffer
|
3
|
+
# @private
|
3
4
|
class SQLite3_Database < Buffer
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
5
|
+
include Quoter
|
6
|
+
|
7
|
+
def chunk
|
8
|
+
return false if rows.empty?
|
9
|
+
row = rows.shift
|
10
|
+
%{
|
11
|
+
INSERT OR IGNORE INTO "#{table_name}" (#{row.columns_sql}) VALUES (#{row.values_sql});
|
12
|
+
UPDATE "#{table_name}" SET #{row.set_sql} WHERE #{row.where_sql}
|
13
|
+
}
|
12
14
|
end
|
13
15
|
|
14
16
|
def execute(sql)
|
15
17
|
connection.execute_batch sql
|
16
18
|
end
|
17
19
|
|
18
|
-
def
|
19
|
-
|
20
|
+
def quote_string(v)
|
21
|
+
SINGLE_QUOTE + SQLite3::Database.quote(v) + SINGLE_QUOTE
|
20
22
|
end
|
21
23
|
|
22
|
-
def
|
23
|
-
|
24
|
+
def quote_binary(v)
|
25
|
+
X_AND_SINGLE_QUOTE + v.unpack("H*")[0] + SINGLE_QUOTE
|
24
26
|
end
|
25
27
|
|
26
|
-
|
27
|
-
|
28
|
-
def quote_value(v)
|
29
|
-
case v
|
30
|
-
when NilClass
|
31
|
-
'NULL'
|
32
|
-
when String, Symbol
|
33
|
-
SINGLE_QUOTE + SQLite3::Database.quote(v.to_s) + SINGLE_QUOTE
|
34
|
-
else
|
35
|
-
v
|
36
|
-
end
|
28
|
+
def quote_time(v)
|
29
|
+
quote_string [v.strftime(ISO8601_DATETIME), sprintf(USEC_SPRINTF, v.usec)].join('.')
|
37
30
|
end
|
38
31
|
|
39
32
|
def quote_ident(k)
|
40
33
|
DOUBLE_QUOTE + SQLite3::Database.quote(k.to_s) + DOUBLE_QUOTE
|
41
34
|
end
|
35
|
+
|
36
|
+
def quote_boolean(v)
|
37
|
+
s = v ? 't' : 'f'
|
38
|
+
quote_string s
|
39
|
+
end
|
40
|
+
|
41
|
+
def quote_big_decimal(v)
|
42
|
+
v.to_f
|
43
|
+
end
|
42
44
|
end
|
43
45
|
end
|
44
46
|
end
|
data/lib/upsert/quoter.rb
CHANGED
@@ -1,7 +1,35 @@
|
|
1
1
|
class Upsert
|
2
|
+
# @private
|
2
3
|
module Quoter
|
4
|
+
ISO8601_DATE = '%F'
|
5
|
+
|
6
|
+
def quote_value(v)
|
7
|
+
case v
|
8
|
+
when NilClass
|
9
|
+
'NULL'
|
10
|
+
when Upsert::Binary
|
11
|
+
quote_binary v # must be defined by base
|
12
|
+
when String
|
13
|
+
quote_string v # must be defined by base
|
14
|
+
when TrueClass, FalseClass
|
15
|
+
quote_boolean v
|
16
|
+
when BigDecimal
|
17
|
+
quote_big_decimal v
|
18
|
+
when Numeric
|
19
|
+
v
|
20
|
+
when Symbol
|
21
|
+
quote_string v.to_s
|
22
|
+
when Time, DateTime
|
23
|
+
quote_time v # must be defined by base
|
24
|
+
when Date
|
25
|
+
quote_string v.strftime(ISO8601_DATE)
|
26
|
+
else
|
27
|
+
raise "not sure how to quote #{v.class}: #{v.inspect}"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
3
31
|
def quote_idents(idents)
|
4
|
-
idents.map { |k| quote_ident(k) }.join(',')
|
32
|
+
idents.map { |k| quote_ident(k) }.join(',') # must be defined by base
|
5
33
|
end
|
6
34
|
|
7
35
|
def quote_values(values)
|
data/lib/upsert/row.rb
CHANGED
@@ -1,36 +1,52 @@
|
|
1
1
|
class Upsert
|
2
|
+
# @private
|
2
3
|
class Row
|
4
|
+
attr_reader :buffer
|
3
5
|
attr_reader :selector
|
4
6
|
attr_reader :document
|
5
7
|
|
6
|
-
def initialize(selector, document)
|
8
|
+
def initialize(buffer, selector, document)
|
9
|
+
@buffer = buffer
|
7
10
|
@selector = selector
|
8
11
|
@document = document
|
9
12
|
end
|
10
13
|
|
11
14
|
def columns
|
12
|
-
@columns ||= (selector.keys+document.keys).uniq
|
15
|
+
@columns ||= (selector.keys + document.keys).uniq
|
16
|
+
end
|
17
|
+
|
18
|
+
def values_sql_length
|
19
|
+
@values_sql_length ||= pairs.inject(0) { |sum, (_, v)| sum + buffer.quoted_value_length(v) }
|
20
|
+
end
|
21
|
+
|
22
|
+
def values_sql
|
23
|
+
buffer.quote_values pairs.map { |_, v| v }
|
24
|
+
end
|
25
|
+
|
26
|
+
def columns_sql
|
27
|
+
buffer.quote_idents columns
|
28
|
+
end
|
29
|
+
|
30
|
+
def where_sql
|
31
|
+
buffer.quote_pairs selector
|
32
|
+
end
|
33
|
+
|
34
|
+
def set_sql
|
35
|
+
buffer.quote_pairs pairs
|
13
36
|
end
|
14
37
|
|
15
38
|
def pairs
|
16
39
|
@pairs ||= columns.map do |k|
|
17
|
-
value = if
|
18
|
-
|
19
|
-
else
|
40
|
+
value = if document.has_key?(k)
|
41
|
+
# prefer the document so that you can change rows
|
20
42
|
document[k]
|
43
|
+
else
|
44
|
+
selector[k]
|
21
45
|
end
|
22
46
|
[ k, value ]
|
23
47
|
end
|
24
48
|
end
|
25
49
|
|
26
|
-
def inserts
|
27
|
-
@inserts ||= pairs.map { |_, v| v }
|
28
|
-
end
|
29
|
-
|
30
|
-
def updates
|
31
|
-
@updates ||= pairs.reject { |k, _| selector.has_key?(k) }
|
32
|
-
end
|
33
|
-
|
34
50
|
def to_hash
|
35
51
|
@to_hash ||= pairs.inject({}) do |memo, (k, v)|
|
36
52
|
memo[k.to_s] = v
|
data/lib/upsert/version.rb
CHANGED
data/test/helper.rb
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
require 'rubygems'
|
2
2
|
require 'bundler/setup'
|
3
|
+
require 'zlib'
|
4
|
+
require 'benchmark'
|
5
|
+
require 'faker'
|
3
6
|
require 'minitest/spec'
|
4
7
|
require 'minitest/autorun'
|
5
8
|
require 'minitest/reporters'
|
@@ -7,6 +10,7 @@ MiniTest::Unit.runner = MiniTest::SuiteRunner.new
|
|
7
10
|
MiniTest::Unit.runner.reporters << MiniTest::Reporters::SpecReporter.new
|
8
11
|
|
9
12
|
require 'active_record'
|
13
|
+
require 'activerecord-import'
|
10
14
|
require 'active_record_inline_schema'
|
11
15
|
|
12
16
|
# require 'logger'
|
@@ -14,9 +18,16 @@ require 'active_record_inline_schema'
|
|
14
18
|
# ActiveRecord::Base.logger.level = Logger::DEBUG
|
15
19
|
|
16
20
|
class Pet < ActiveRecord::Base
|
17
|
-
self.primary_key = 'name'
|
18
21
|
col :name
|
19
22
|
col :gender
|
23
|
+
col :good, :type => :boolean
|
24
|
+
col :lovability, :type => :float
|
25
|
+
col :morning_walk_time, :type => :datetime
|
26
|
+
col :zipped_biography, :type => :binary
|
27
|
+
col :tag_number, :type => :integer
|
28
|
+
col :birthday, :type => :date
|
29
|
+
col :home_address, :type => :text
|
30
|
+
add_index :name, :unique => true
|
20
31
|
end
|
21
32
|
|
22
33
|
require 'upsert'
|
@@ -26,6 +37,47 @@ MiniTest::Spec.class_eval do
|
|
26
37
|
@shared_examples ||= {}
|
27
38
|
end
|
28
39
|
|
40
|
+
def lotsa_records
|
41
|
+
@records ||= begin
|
42
|
+
memo = []
|
43
|
+
names = []
|
44
|
+
50.times do
|
45
|
+
names << Faker::Name.name
|
46
|
+
end
|
47
|
+
200.times do
|
48
|
+
selector = ActiveSupport::OrderedHash.new
|
49
|
+
selector[:name] = names.sample(1).first
|
50
|
+
document = {
|
51
|
+
:lovability => BigDecimal.new(rand(1e11), 2),
|
52
|
+
:tag_number => rand(1e8),
|
53
|
+
:good => true,
|
54
|
+
:birthday => Time.at(rand * Time.now.to_i).to_date,
|
55
|
+
:morning_walk_time => Time.at(rand * Time.now.to_i),
|
56
|
+
:home_address => Faker::Address.street_address,
|
57
|
+
:zipped_biography => Upsert.binary(Zlib::Deflate.deflate(Faker::Lorem.paragraphs(10).join("\n\n"), Zlib::BEST_SPEED))
|
58
|
+
}
|
59
|
+
memo << [selector, document]
|
60
|
+
end
|
61
|
+
memo
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def assert_same_result(records, &blk)
|
66
|
+
blk.call(records)
|
67
|
+
ref1 = Pet.order(:name).all.map { |pet| pet.attributes.except('id') }
|
68
|
+
|
69
|
+
Pet.delete_all
|
70
|
+
|
71
|
+
upsert = Upsert.new connection, :pets
|
72
|
+
upsert.multi do |xxx|
|
73
|
+
records.each do |selector, document|
|
74
|
+
xxx.row(selector, document)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
ref2 = Pet.order(:name).all.map { |pet| pet.attributes.except('id') }
|
78
|
+
ref2.must_equal ref1
|
79
|
+
end
|
80
|
+
|
29
81
|
def assert_creates(model, expected_records)
|
30
82
|
expected_records.each do |conditions|
|
31
83
|
model.where(conditions).count.must_equal 0
|
@@ -36,6 +88,29 @@ MiniTest::Spec.class_eval do
|
|
36
88
|
end
|
37
89
|
end
|
38
90
|
|
91
|
+
def assert_faster_than(competition, records, &blk)
|
92
|
+
# dry run
|
93
|
+
blk.call records
|
94
|
+
Pet.delete_all
|
95
|
+
sleep 1
|
96
|
+
# --
|
97
|
+
|
98
|
+
ar_time = Benchmark.realtime { blk.call(records) }
|
99
|
+
|
100
|
+
Pet.delete_all
|
101
|
+
sleep 1
|
102
|
+
|
103
|
+
upsert_time = Benchmark.realtime do
|
104
|
+
upsert = Upsert.new connection, :pets
|
105
|
+
upsert.multi do |xxx|
|
106
|
+
records.each do |selector, document|
|
107
|
+
xxx.row(selector, document)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
upsert_time.must_be :<, ar_time
|
112
|
+
$stderr.puts " Upsert was #{((ar_time - upsert_time) / ar_time * 100).round}% faster than #{competition}"
|
113
|
+
end
|
39
114
|
end
|
40
115
|
|
41
116
|
module MiniTest::Spec::SharedExamples
|
@@ -43,7 +118,7 @@ module MiniTest::Spec::SharedExamples
|
|
43
118
|
MiniTest::Spec.shared_examples[desc] = block
|
44
119
|
end
|
45
120
|
|
46
|
-
def
|
121
|
+
def it_also(desc)
|
47
122
|
self.instance_eval do
|
48
123
|
MiniTest::Spec.shared_examples[desc].call
|
49
124
|
end
|
@@ -51,4 +126,6 @@ module MiniTest::Spec::SharedExamples
|
|
51
126
|
end
|
52
127
|
|
53
128
|
Object.class_eval { include(MiniTest::Spec::SharedExamples) }
|
54
|
-
|
129
|
+
Dir[File.expand_path("../shared/*.rb", __FILE__)].each do |path|
|
130
|
+
require path
|
131
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
shared_examples_for 'supports binary upserts' do
|
2
|
+
describe 'binary' do
|
3
|
+
before do
|
4
|
+
@fakes = []
|
5
|
+
10.times do
|
6
|
+
@fakes << [Faker::Name.name, Faker::Lorem.paragraphs(10).join("\n\n")]
|
7
|
+
end
|
8
|
+
end
|
9
|
+
it "saves binary one by one" do
|
10
|
+
@fakes.each do |name, biography|
|
11
|
+
zipped_biography = Zlib::Deflate.deflate biography
|
12
|
+
upsert = Upsert.new connection, :pets
|
13
|
+
assert_creates(Pet, [{:name => name, :zipped_biography => zipped_biography}]) do
|
14
|
+
upsert.row({:name => name}, {:zipped_biography => Upsert.binary(zipped_biography)})
|
15
|
+
end
|
16
|
+
Zlib::Inflate.inflate(Pet.find_by_name(name).zipped_biography).must_equal biography
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
shared_examples_for 'is just as correct as other ways' do
|
2
|
+
describe :correctness do
|
3
|
+
describe 'compared to native ActiveRecord' do
|
4
|
+
it "is as correct as than new/set/save" do
|
5
|
+
assert_same_result lotsa_records do |records|
|
6
|
+
records.each do |selector, document|
|
7
|
+
if pet = Pet.where(selector).first
|
8
|
+
pet.update_attributes document, :without_protection => true
|
9
|
+
else
|
10
|
+
pet = Pet.new
|
11
|
+
selector.each do |k, v|
|
12
|
+
pet.send "#{k}=", v
|
13
|
+
end
|
14
|
+
document.each do |k, v|
|
15
|
+
pet.send "#{k}=", v
|
16
|
+
end
|
17
|
+
pet.save!
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
it "is as correct as than find_or_create + update_attributes" do
|
23
|
+
assert_same_result lotsa_records do |records|
|
24
|
+
dynamic_method = nil
|
25
|
+
records.each do |selector, document|
|
26
|
+
dynamic_method ||= "find_or_create_by_#{selector.keys.join('_or_')}"
|
27
|
+
pet = Pet.send(dynamic_method, *selector.values)
|
28
|
+
pet.update_attributes document, :without_protection => true
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
it "is as correct as than create + rescue/find/update" do
|
33
|
+
assert_same_result lotsa_records do |records|
|
34
|
+
dynamic_method = nil
|
35
|
+
records.each do |selector, document|
|
36
|
+
dynamic_method ||= "find_or_create_by_#{selector.keys.join('_or_')}"
|
37
|
+
begin
|
38
|
+
Pet.create selector.merge(document), :without_protection => true
|
39
|
+
rescue
|
40
|
+
pet = Pet.send(dynamic_method, *selector.values)
|
41
|
+
pet.update_attributes document, :without_protection => true
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
shared_examples_for
|
1
|
+
shared_examples_for 'is a database with an upsert trick' do
|
2
2
|
describe :row do
|
3
3
|
it "works for a single row (base case)" do
|
4
4
|
upsert = Upsert.new connection, :pets
|
@@ -52,26 +52,26 @@ shared_examples_for :database do
|
|
52
52
|
it "works for multiple rows (base case)" do
|
53
53
|
upsert = Upsert.new connection, :pets
|
54
54
|
assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
|
55
|
-
upsert.multi do
|
56
|
-
row({:name => 'Jerry'}, :gender => 'male')
|
55
|
+
upsert.multi do |xxx|
|
56
|
+
xxx.row({:name => 'Jerry'}, :gender => 'male')
|
57
57
|
end
|
58
58
|
end
|
59
59
|
end
|
60
60
|
it "works for multiple rows (not changing anything)" do
|
61
61
|
upsert = Upsert.new connection, :pets
|
62
62
|
assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
|
63
|
-
upsert.multi do
|
64
|
-
row({:name => 'Jerry'}, :gender => 'male')
|
65
|
-
row({:name => 'Jerry'}, :gender => 'male')
|
63
|
+
upsert.multi do |xxx|
|
64
|
+
xxx.row({:name => 'Jerry'}, :gender => 'male')
|
65
|
+
xxx.row({:name => 'Jerry'}, :gender => 'male')
|
66
66
|
end
|
67
67
|
end
|
68
68
|
end
|
69
69
|
it "works for multiple rows (changing something)" do
|
70
70
|
upsert = Upsert.new connection, :pets
|
71
71
|
assert_creates(Pet, [{:name => 'Jerry', :gender => 'neutered'}]) do
|
72
|
-
upsert.multi do
|
73
|
-
row({:name => 'Jerry'}, :gender => 'male')
|
74
|
-
row({:name => 'Jerry'}, :gender => 'neutered')
|
72
|
+
upsert.multi do |xxx|
|
73
|
+
xxx.row({:name => 'Jerry'}, :gender => 'male')
|
74
|
+
xxx.row({:name => 'Jerry'}, :gender => 'neutered')
|
75
75
|
end
|
76
76
|
end
|
77
77
|
Pet.where(:gender => 'male').count.must_equal 0
|