upsert 0.0.1 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,10 +1,34 @@
1
1
  class Upsert
2
2
  class Buffer
3
3
  class PG_Connection < Buffer
4
+ # @private
5
+ # activerecord-3.2.5/lib/active_record/connection_adapters/postgresql_adapter.rb#column_definitions
4
6
  class ColumnDefinition
5
7
  class << self
8
+ def auto_increment_primary_key(connection, table_name)
9
+ res = connection.exec <<-EOS
10
+ SELECT attr.attname, seq.relname
11
+ FROM pg_class seq,
12
+ pg_attribute attr,
13
+ pg_depend dep,
14
+ pg_namespace name,
15
+ pg_constraint cons
16
+ WHERE seq.oid = dep.objid
17
+ AND seq.relkind = 'S'
18
+ AND attr.attrelid = dep.refobjid
19
+ AND attr.attnum = dep.refobjsubid
20
+ AND attr.attrelid = cons.conrelid
21
+ AND attr.attnum = cons.conkey[1]
22
+ AND cons.contype = 'p'
23
+ AND dep.refobjid = '#{connection.quote_ident(table_name.to_s)}'::regclass
24
+ EOS
25
+ if hit = res.first
26
+ hit['attname']
27
+ end
28
+ end
29
+
6
30
  def all(connection, table_name)
7
- # activerecord-3.2.5/lib/active_record/connection_adapters/postgresql_adapter.rb#column_definitions
31
+ auto_increment_primary_key = auto_increment_primary_key(connection, table_name)
8
32
  res = connection.exec <<-EOS
9
33
  SELECT a.attname AS name, format_type(a.atttypid, a.atttypmod) AS sql_type, d.adsrc AS default
10
34
  FROM pg_attribute a LEFT JOIN pg_attrdef d
@@ -13,7 +37,9 @@ SELECT a.attname AS name, format_type(a.atttypid, a.atttypmod) AS sql_type, d.ad
13
37
  AND a.attnum > 0 AND NOT a.attisdropped
14
38
  ORDER BY a.attnum
15
39
  EOS
16
- res.map do |row|
40
+ res.reject do |row|
41
+ row['name'] == auto_increment_primary_key
42
+ end.map do |row|
17
43
  new row['name'], row['sql_type'], row['default']
18
44
  end
19
45
  end
@@ -1,44 +1,46 @@
1
1
  class Upsert
2
2
  class Buffer
3
+ # @private
3
4
  class SQLite3_Database < Buffer
4
- def compose(targets)
5
- target = targets.first
6
- parts = []
7
- parts << %{ INSERT OR IGNORE INTO "#{table_name}" (#{quote_idents(target.columns)}) VALUES (#{quote_values(target.inserts)}) }
8
- if target.updates.length > 0
9
- parts << %{ UPDATE "#{table_name}" SET #{quote_pairs(target.updates)} WHERE #{quote_pairs(target.selector)} }
10
- end
11
- parts.join(';')
5
+ include Quoter
6
+
7
+ def chunk
8
+ return false if rows.empty?
9
+ row = rows.shift
10
+ %{
11
+ INSERT OR IGNORE INTO "#{table_name}" (#{row.columns_sql}) VALUES (#{row.values_sql});
12
+ UPDATE "#{table_name}" SET #{row.set_sql} WHERE #{row.where_sql}
13
+ }
12
14
  end
13
15
 
14
16
  def execute(sql)
15
17
  connection.execute_batch sql
16
18
  end
17
19
 
18
- def max_targets
19
- 1
20
+ def quote_string(v)
21
+ SINGLE_QUOTE + SQLite3::Database.quote(v) + SINGLE_QUOTE
20
22
  end
21
23
 
22
- def max_length
23
- INFINITY
24
+ def quote_binary(v)
25
+ X_AND_SINGLE_QUOTE + v.unpack("H*")[0] + SINGLE_QUOTE
24
26
  end
25
27
 
26
- include Quoter
27
-
28
- def quote_value(v)
29
- case v
30
- when NilClass
31
- 'NULL'
32
- when String, Symbol
33
- SINGLE_QUOTE + SQLite3::Database.quote(v.to_s) + SINGLE_QUOTE
34
- else
35
- v
36
- end
28
+ def quote_time(v)
29
+ quote_string [v.strftime(ISO8601_DATETIME), sprintf(USEC_SPRINTF, v.usec)].join('.')
37
30
  end
38
31
 
39
32
  def quote_ident(k)
40
33
  DOUBLE_QUOTE + SQLite3::Database.quote(k.to_s) + DOUBLE_QUOTE
41
34
  end
35
+
36
+ def quote_boolean(v)
37
+ s = v ? 't' : 'f'
38
+ quote_string s
39
+ end
40
+
41
+ def quote_big_decimal(v)
42
+ v.to_f
43
+ end
42
44
  end
43
45
  end
44
46
  end
data/lib/upsert/quoter.rb CHANGED
@@ -1,7 +1,35 @@
1
1
  class Upsert
2
+ # @private
2
3
  module Quoter
4
+ ISO8601_DATE = '%F'
5
+
6
+ def quote_value(v)
7
+ case v
8
+ when NilClass
9
+ 'NULL'
10
+ when Upsert::Binary
11
+ quote_binary v # must be defined by base
12
+ when String
13
+ quote_string v # must be defined by base
14
+ when TrueClass, FalseClass
15
+ quote_boolean v
16
+ when BigDecimal
17
+ quote_big_decimal v
18
+ when Numeric
19
+ v
20
+ when Symbol
21
+ quote_string v.to_s
22
+ when Time, DateTime
23
+ quote_time v # must be defined by base
24
+ when Date
25
+ quote_string v.strftime(ISO8601_DATE)
26
+ else
27
+ raise "not sure how to quote #{v.class}: #{v.inspect}"
28
+ end
29
+ end
30
+
3
31
  def quote_idents(idents)
4
- idents.map { |k| quote_ident(k) }.join(',')
32
+ idents.map { |k| quote_ident(k) }.join(',') # must be defined by base
5
33
  end
6
34
 
7
35
  def quote_values(values)
data/lib/upsert/row.rb CHANGED
@@ -1,36 +1,52 @@
1
1
  class Upsert
2
+ # @private
2
3
  class Row
4
+ attr_reader :buffer
3
5
  attr_reader :selector
4
6
  attr_reader :document
5
7
 
6
- def initialize(selector, document)
8
+ def initialize(buffer, selector, document)
9
+ @buffer = buffer
7
10
  @selector = selector
8
11
  @document = document
9
12
  end
10
13
 
11
14
  def columns
12
- @columns ||= (selector.keys+document.keys).uniq
15
+ @columns ||= (selector.keys + document.keys).uniq
16
+ end
17
+
18
+ def values_sql_length
19
+ @values_sql_length ||= pairs.inject(0) { |sum, (_, v)| sum + buffer.quoted_value_length(v) }
20
+ end
21
+
22
+ def values_sql
23
+ buffer.quote_values pairs.map { |_, v| v }
24
+ end
25
+
26
+ def columns_sql
27
+ buffer.quote_idents columns
28
+ end
29
+
30
+ def where_sql
31
+ buffer.quote_pairs selector
32
+ end
33
+
34
+ def set_sql
35
+ buffer.quote_pairs pairs
13
36
  end
14
37
 
15
38
  def pairs
16
39
  @pairs ||= columns.map do |k|
17
- value = if selector.has_key?(k)
18
- selector[k]
19
- else
40
+ value = if document.has_key?(k)
41
+ # prefer the document so that you can change rows
20
42
  document[k]
43
+ else
44
+ selector[k]
21
45
  end
22
46
  [ k, value ]
23
47
  end
24
48
  end
25
49
 
26
- def inserts
27
- @inserts ||= pairs.map { |_, v| v }
28
- end
29
-
30
- def updates
31
- @updates ||= pairs.reject { |k, _| selector.has_key?(k) }
32
- end
33
-
34
50
  def to_hash
35
51
  @to_hash ||= pairs.inject({}) do |memo, (k, v)|
36
52
  memo[k.to_s] = v
@@ -1,3 +1,3 @@
1
1
  class Upsert
2
- VERSION = "0.0.1"
2
+ VERSION = "0.1.0"
3
3
  end
data/test/helper.rb CHANGED
@@ -1,5 +1,8 @@
1
1
  require 'rubygems'
2
2
  require 'bundler/setup'
3
+ require 'zlib'
4
+ require 'benchmark'
5
+ require 'faker'
3
6
  require 'minitest/spec'
4
7
  require 'minitest/autorun'
5
8
  require 'minitest/reporters'
@@ -7,6 +10,7 @@ MiniTest::Unit.runner = MiniTest::SuiteRunner.new
7
10
  MiniTest::Unit.runner.reporters << MiniTest::Reporters::SpecReporter.new
8
11
 
9
12
  require 'active_record'
13
+ require 'activerecord-import'
10
14
  require 'active_record_inline_schema'
11
15
 
12
16
  # require 'logger'
@@ -14,9 +18,16 @@ require 'active_record_inline_schema'
14
18
  # ActiveRecord::Base.logger.level = Logger::DEBUG
15
19
 
16
20
  class Pet < ActiveRecord::Base
17
- self.primary_key = 'name'
18
21
  col :name
19
22
  col :gender
23
+ col :good, :type => :boolean
24
+ col :lovability, :type => :float
25
+ col :morning_walk_time, :type => :datetime
26
+ col :zipped_biography, :type => :binary
27
+ col :tag_number, :type => :integer
28
+ col :birthday, :type => :date
29
+ col :home_address, :type => :text
30
+ add_index :name, :unique => true
20
31
  end
21
32
 
22
33
  require 'upsert'
@@ -26,6 +37,47 @@ MiniTest::Spec.class_eval do
26
37
  @shared_examples ||= {}
27
38
  end
28
39
 
40
+ def lotsa_records
41
+ @records ||= begin
42
+ memo = []
43
+ names = []
44
+ 50.times do
45
+ names << Faker::Name.name
46
+ end
47
+ 200.times do
48
+ selector = ActiveSupport::OrderedHash.new
49
+ selector[:name] = names.sample(1).first
50
+ document = {
51
+ :lovability => BigDecimal.new(rand(1e11), 2),
52
+ :tag_number => rand(1e8),
53
+ :good => true,
54
+ :birthday => Time.at(rand * Time.now.to_i).to_date,
55
+ :morning_walk_time => Time.at(rand * Time.now.to_i),
56
+ :home_address => Faker::Address.street_address,
57
+ :zipped_biography => Upsert.binary(Zlib::Deflate.deflate(Faker::Lorem.paragraphs(10).join("\n\n"), Zlib::BEST_SPEED))
58
+ }
59
+ memo << [selector, document]
60
+ end
61
+ memo
62
+ end
63
+ end
64
+
65
+ def assert_same_result(records, &blk)
66
+ blk.call(records)
67
+ ref1 = Pet.order(:name).all.map { |pet| pet.attributes.except('id') }
68
+
69
+ Pet.delete_all
70
+
71
+ upsert = Upsert.new connection, :pets
72
+ upsert.multi do |xxx|
73
+ records.each do |selector, document|
74
+ xxx.row(selector, document)
75
+ end
76
+ end
77
+ ref2 = Pet.order(:name).all.map { |pet| pet.attributes.except('id') }
78
+ ref2.must_equal ref1
79
+ end
80
+
29
81
  def assert_creates(model, expected_records)
30
82
  expected_records.each do |conditions|
31
83
  model.where(conditions).count.must_equal 0
@@ -36,6 +88,29 @@ MiniTest::Spec.class_eval do
36
88
  end
37
89
  end
38
90
 
91
+ def assert_faster_than(competition, records, &blk)
92
+ # dry run
93
+ blk.call records
94
+ Pet.delete_all
95
+ sleep 1
96
+ # --
97
+
98
+ ar_time = Benchmark.realtime { blk.call(records) }
99
+
100
+ Pet.delete_all
101
+ sleep 1
102
+
103
+ upsert_time = Benchmark.realtime do
104
+ upsert = Upsert.new connection, :pets
105
+ upsert.multi do |xxx|
106
+ records.each do |selector, document|
107
+ xxx.row(selector, document)
108
+ end
109
+ end
110
+ end
111
+ upsert_time.must_be :<, ar_time
112
+ $stderr.puts " Upsert was #{((ar_time - upsert_time) / ar_time * 100).round}% faster than #{competition}"
113
+ end
39
114
  end
40
115
 
41
116
  module MiniTest::Spec::SharedExamples
@@ -43,7 +118,7 @@ module MiniTest::Spec::SharedExamples
43
118
  MiniTest::Spec.shared_examples[desc] = block
44
119
  end
45
120
 
46
- def it_behaves_like(desc)
121
+ def it_also(desc)
47
122
  self.instance_eval do
48
123
  MiniTest::Spec.shared_examples[desc].call
49
124
  end
@@ -51,4 +126,6 @@ module MiniTest::Spec::SharedExamples
51
126
  end
52
127
 
53
128
  Object.class_eval { include(MiniTest::Spec::SharedExamples) }
54
- require 'shared_examples'
129
+ Dir[File.expand_path("../shared/*.rb", __FILE__)].each do |path|
130
+ require path
131
+ end
@@ -0,0 +1,20 @@
1
+ shared_examples_for 'supports binary upserts' do
2
+ describe 'binary' do
3
+ before do
4
+ @fakes = []
5
+ 10.times do
6
+ @fakes << [Faker::Name.name, Faker::Lorem.paragraphs(10).join("\n\n")]
7
+ end
8
+ end
9
+ it "saves binary one by one" do
10
+ @fakes.each do |name, biography|
11
+ zipped_biography = Zlib::Deflate.deflate biography
12
+ upsert = Upsert.new connection, :pets
13
+ assert_creates(Pet, [{:name => name, :zipped_biography => zipped_biography}]) do
14
+ upsert.row({:name => name}, {:zipped_biography => Upsert.binary(zipped_biography)})
15
+ end
16
+ Zlib::Inflate.inflate(Pet.find_by_name(name).zipped_biography).must_equal biography
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,48 @@
1
+ shared_examples_for 'is just as correct as other ways' do
2
+ describe :correctness do
3
+ describe 'compared to native ActiveRecord' do
4
+ it "is as correct as than new/set/save" do
5
+ assert_same_result lotsa_records do |records|
6
+ records.each do |selector, document|
7
+ if pet = Pet.where(selector).first
8
+ pet.update_attributes document, :without_protection => true
9
+ else
10
+ pet = Pet.new
11
+ selector.each do |k, v|
12
+ pet.send "#{k}=", v
13
+ end
14
+ document.each do |k, v|
15
+ pet.send "#{k}=", v
16
+ end
17
+ pet.save!
18
+ end
19
+ end
20
+ end
21
+ end
22
+ it "is as correct as than find_or_create + update_attributes" do
23
+ assert_same_result lotsa_records do |records|
24
+ dynamic_method = nil
25
+ records.each do |selector, document|
26
+ dynamic_method ||= "find_or_create_by_#{selector.keys.join('_or_')}"
27
+ pet = Pet.send(dynamic_method, *selector.values)
28
+ pet.update_attributes document, :without_protection => true
29
+ end
30
+ end
31
+ end
32
+ it "is as correct as than create + rescue/find/update" do
33
+ assert_same_result lotsa_records do |records|
34
+ dynamic_method = nil
35
+ records.each do |selector, document|
36
+ dynamic_method ||= "find_or_create_by_#{selector.keys.join('_or_')}"
37
+ begin
38
+ Pet.create selector.merge(document), :without_protection => true
39
+ rescue
40
+ pet = Pet.send(dynamic_method, *selector.values)
41
+ pet.update_attributes document, :without_protection => true
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
@@ -1,4 +1,4 @@
1
- shared_examples_for :database do
1
+ shared_examples_for 'is a database with an upsert trick' do
2
2
  describe :row do
3
3
  it "works for a single row (base case)" do
4
4
  upsert = Upsert.new connection, :pets
@@ -52,26 +52,26 @@ shared_examples_for :database do
52
52
  it "works for multiple rows (base case)" do
53
53
  upsert = Upsert.new connection, :pets
54
54
  assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
55
- upsert.multi do
56
- row({:name => 'Jerry'}, :gender => 'male')
55
+ upsert.multi do |xxx|
56
+ xxx.row({:name => 'Jerry'}, :gender => 'male')
57
57
  end
58
58
  end
59
59
  end
60
60
  it "works for multiple rows (not changing anything)" do
61
61
  upsert = Upsert.new connection, :pets
62
62
  assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
63
- upsert.multi do
64
- row({:name => 'Jerry'}, :gender => 'male')
65
- row({:name => 'Jerry'}, :gender => 'male')
63
+ upsert.multi do |xxx|
64
+ xxx.row({:name => 'Jerry'}, :gender => 'male')
65
+ xxx.row({:name => 'Jerry'}, :gender => 'male')
66
66
  end
67
67
  end
68
68
  end
69
69
  it "works for multiple rows (changing something)" do
70
70
  upsert = Upsert.new connection, :pets
71
71
  assert_creates(Pet, [{:name => 'Jerry', :gender => 'neutered'}]) do
72
- upsert.multi do
73
- row({:name => 'Jerry'}, :gender => 'male')
74
- row({:name => 'Jerry'}, :gender => 'neutered')
72
+ upsert.multi do |xxx|
73
+ xxx.row({:name => 'Jerry'}, :gender => 'male')
74
+ xxx.row({:name => 'Jerry'}, :gender => 'neutered')
75
75
  end
76
76
  end
77
77
  Pet.where(:gender => 'male').count.must_equal 0