upsert 0.0.1 → 0.1.0

data/lib/upsert/buffer/pg_connection/column_definition.rb

@@ -1,10 +1,34 @@
 class Upsert
   class Buffer
     class PG_Connection < Buffer
+      # @private
+      # activerecord-3.2.5/lib/active_record/connection_adapters/postgresql_adapter.rb#column_definitions
       class ColumnDefinition
         class << self
+          def auto_increment_primary_key(connection, table_name)
+            res = connection.exec <<-EOS
+              SELECT attr.attname, seq.relname
+              FROM pg_class      seq,
+                   pg_attribute  attr,
+                   pg_depend     dep,
+                   pg_namespace  name,
+                   pg_constraint cons
+              WHERE seq.oid           = dep.objid
+                AND seq.relkind       = 'S'
+                AND attr.attrelid     = dep.refobjid
+                AND attr.attnum       = dep.refobjsubid
+                AND attr.attrelid     = cons.conrelid
+                AND attr.attnum       = cons.conkey[1]
+                AND cons.contype      = 'p'
+                AND dep.refobjid      = '#{connection.quote_ident(table_name.to_s)}'::regclass
+EOS
+            if hit = res.first
+              hit['attname']
+            end
+          end
+
           def all(connection, table_name)
-            # activerecord-3.2.5/lib/active_record/connection_adapters/postgresql_adapter.rb#column_definitions
+            auto_increment_primary_key = auto_increment_primary_key(connection, table_name)
             res = connection.exec <<-EOS
 SELECT a.attname AS name, format_type(a.atttypid, a.atttypmod) AS sql_type, d.adsrc AS default
   FROM pg_attribute a LEFT JOIN pg_attrdef d
@@ -13,7 +37,9 @@ SELECT a.attname AS name, format_type(a.atttypid, a.atttypmod) AS sql_type, d.ad
    AND a.attnum > 0 AND NOT a.attisdropped
  ORDER BY a.attnum
  EOS
-            res.map do |row|
+            res.reject do |row|
+              row['name'] == auto_increment_primary_key
+            end.map do |row|
               new row['name'], row['sql_type'], row['default']
             end
           end

data/lib/upsert/buffer/sqlite3_database.rb

@@ -1,44 +1,46 @@
 class Upsert
   class Buffer
+    # @private
     class SQLite3_Database < Buffer
-      def compose(targets)
-        target = targets.first
-        parts = []
-        parts << %{ INSERT OR IGNORE INTO "#{table_name}" (#{quote_idents(target.columns)}) VALUES (#{quote_values(target.inserts)}) }
-        if target.updates.length > 0
-          parts << %{ UPDATE "#{table_name}" SET #{quote_pairs(target.updates)} WHERE #{quote_pairs(target.selector)} }
-        end
-        parts.join(';')
+      include Quoter
+
+      def chunk
+        return false if rows.empty?
+        row = rows.shift
+        %{
+          INSERT OR IGNORE INTO "#{table_name}" (#{row.columns_sql}) VALUES (#{row.values_sql});
+          UPDATE "#{table_name}" SET #{row.set_sql} WHERE #{row.where_sql}
+        }
       end
 
       def execute(sql)
         connection.execute_batch sql
       end
 
-      def max_targets
-        1
+      def quote_string(v)
+        SINGLE_QUOTE + SQLite3::Database.quote(v) + SINGLE_QUOTE
       end
 
-      def max_length
-        INFINITY
+      def quote_binary(v)
+        X_AND_SINGLE_QUOTE + v.unpack("H*")[0] + SINGLE_QUOTE
       end
 
-      include Quoter
-      
-      def quote_value(v)
-        case v
-        when NilClass
-          'NULL'
-        when String, Symbol
-          SINGLE_QUOTE + SQLite3::Database.quote(v.to_s) + SINGLE_QUOTE
-        else
-          v
-        end
+      def quote_time(v)
+        quote_string [v.strftime(ISO8601_DATETIME), sprintf(USEC_SPRINTF, v.usec)].join('.')
       end
       
       def quote_ident(k)
         DOUBLE_QUOTE + SQLite3::Database.quote(k.to_s) + DOUBLE_QUOTE
       end
+
+      def quote_boolean(v)
+        s = v ? 't' : 'f'
+        quote_string s
+      end
+
+      def quote_big_decimal(v)
+        v.to_f
+      end
     end
   end
 end

data/lib/upsert/quoter.rb

@@ -1,7 +1,35 @@
 class Upsert
+  # @private
   module Quoter
+    ISO8601_DATE = '%F'
+
+    def quote_value(v)
+      case v
+      when NilClass
+        'NULL'
+      when Upsert::Binary
+        quote_binary v # must be defined by base
+      when String
+        quote_string v # must be defined by base
+      when TrueClass, FalseClass
+        quote_boolean v
+      when BigDecimal
+        quote_big_decimal v
+      when Numeric
+        v
+      when Symbol
+        quote_string v.to_s
+      when Time, DateTime
+        quote_time v # must be defined by base
+      when Date
+        quote_string v.strftime(ISO8601_DATE)
+      else
+        raise "not sure how to quote #{v.class}: #{v.inspect}"
+      end
+    end
+
     def quote_idents(idents)
-      idents.map { |k| quote_ident(k) }.join(',')
+      idents.map { |k| quote_ident(k) }.join(',') # must be defined by base
     end
 
     def quote_values(values)

data/lib/upsert/row.rb

@@ -1,36 +1,52 @@
 class Upsert
+  # @private
   class Row
+    attr_reader :buffer
     attr_reader :selector
     attr_reader :document
 
-    def initialize(selector, document)
+    def initialize(buffer, selector, document)
+      @buffer = buffer
       @selector = selector
       @document = document
     end
 
     def columns
-      @columns ||= (selector.keys+document.keys).uniq
+      @columns ||= (selector.keys + document.keys).uniq
+    end
+
+    def values_sql_length
+      @values_sql_length ||= pairs.inject(0) { |sum, (_, v)| sum + buffer.quoted_value_length(v) }
+    end
+
+    def values_sql
+      buffer.quote_values pairs.map { |_, v| v }
+    end
+
+    def columns_sql
+      buffer.quote_idents columns
+    end
+
+    def where_sql
+      buffer.quote_pairs selector
+    end
+
+    def set_sql
+      buffer.quote_pairs pairs
     end
 
     def pairs
       @pairs ||= columns.map do |k|
-        value = if selector.has_key?(k)
-          selector[k]
-        else
+        value = if document.has_key?(k)
+          # prefer the document so that you can change rows
           document[k]
+        else
+          selector[k]
         end
         [ k, value ]
       end
     end
 
-    def inserts
-      @inserts ||= pairs.map { |_, v| v }
-    end
-
-    def updates
-      @updates ||= pairs.reject { |k, _| selector.has_key?(k) }
-    end
-    
     def to_hash
       @to_hash ||= pairs.inject({}) do |memo, (k, v)|
         memo[k.to_s] = v

data/lib/upsert/version.rb

@@ -1,3 +1,3 @@
 class Upsert
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/test/helper.rb

@@ -1,5 +1,8 @@
 require 'rubygems'
 require 'bundler/setup'
+require 'zlib'
+require 'benchmark'
+require 'faker'
 require 'minitest/spec'
 require 'minitest/autorun'
 require 'minitest/reporters'
@@ -7,6 +10,7 @@ MiniTest::Unit.runner = MiniTest::SuiteRunner.new
 MiniTest::Unit.runner.reporters << MiniTest::Reporters::SpecReporter.new
 
 require 'active_record'
+require 'activerecord-import'
 require 'active_record_inline_schema'
 
 # require 'logger'
@@ -14,9 +18,16 @@ require 'active_record_inline_schema'
 # ActiveRecord::Base.logger.level = Logger::DEBUG
 
 class Pet < ActiveRecord::Base
-  self.primary_key = 'name'
   col :name
   col :gender
+  col :good, :type => :boolean
+  col :lovability, :type => :float
+  col :morning_walk_time, :type => :datetime
+  col :zipped_biography, :type => :binary
+  col :tag_number, :type => :integer
+  col :birthday, :type => :date
+  col :home_address, :type => :text
+  add_index :name, :unique => true
 end
 
 require 'upsert'
@@ -26,6 +37,47 @@ MiniTest::Spec.class_eval do
     @shared_examples ||= {}
   end
 
+  def lotsa_records
+    @records ||= begin
+      memo = []
+      names = []
+      50.times do
+        names << Faker::Name.name
+      end
+      200.times do
+        selector = ActiveSupport::OrderedHash.new
+        selector[:name] = names.sample(1).first
+        document = {
+          :lovability => BigDecimal.new(rand(1e11), 2),
+          :tag_number => rand(1e8),
+          :good => true,
+          :birthday => Time.at(rand * Time.now.to_i).to_date,
+          :morning_walk_time => Time.at(rand * Time.now.to_i),
+          :home_address => Faker::Address.street_address,
+          :zipped_biography => Upsert.binary(Zlib::Deflate.deflate(Faker::Lorem.paragraphs(10).join("\n\n"), Zlib::BEST_SPEED))
+        }
+        memo << [selector, document]
+      end
+      memo
+    end
+  end
+
+  def assert_same_result(records, &blk)
+    blk.call(records)
+    ref1 = Pet.order(:name).all.map { |pet| pet.attributes.except('id') }
+    
+    Pet.delete_all
+
+    upsert = Upsert.new connection, :pets
+    upsert.multi do |xxx|
+      records.each do |selector, document|
+        xxx.row(selector, document)
+      end
+    end
+    ref2 = Pet.order(:name).all.map { |pet| pet.attributes.except('id') }
+    ref2.must_equal ref1
+  end
+
   def assert_creates(model, expected_records)
     expected_records.each do |conditions|
       model.where(conditions).count.must_equal 0
@@ -36,6 +88,29 @@ MiniTest::Spec.class_eval do
     end
   end
 
+  def assert_faster_than(competition, records, &blk)
+    # dry run
+    blk.call records
+    Pet.delete_all
+    sleep 1
+    # --
+    
+    ar_time = Benchmark.realtime { blk.call(records) }
+
+    Pet.delete_all
+    sleep 1
+
+    upsert_time = Benchmark.realtime do
+      upsert = Upsert.new connection, :pets
+      upsert.multi do |xxx|
+        records.each do |selector, document|
+          xxx.row(selector, document)
+        end
+      end
+    end
+    upsert_time.must_be :<, ar_time
+    $stderr.puts "   Upsert was #{((ar_time - upsert_time) / ar_time * 100).round}% faster than #{competition}"
+  end
 end
 
 module MiniTest::Spec::SharedExamples
@@ -43,7 +118,7 @@ module MiniTest::Spec::SharedExamples
     MiniTest::Spec.shared_examples[desc] = block
   end
 
-  def it_behaves_like(desc)
+  def it_also(desc)
     self.instance_eval do
       MiniTest::Spec.shared_examples[desc].call
     end
@@ -51,4 +126,6 @@ module MiniTest::Spec::SharedExamples
 end
 
 Object.class_eval { include(MiniTest::Spec::SharedExamples) }
-require 'shared_examples'
+Dir[File.expand_path("../shared/*.rb", __FILE__)].each do |path|
+  require path
+end

data/test/shared/binary.rb

@@ -0,0 +1,20 @@
+shared_examples_for 'supports binary upserts' do
+  describe 'binary' do
+    before do
+      @fakes = []
+      10.times do
+        @fakes << [Faker::Name.name, Faker::Lorem.paragraphs(10).join("\n\n")]
+      end
+    end
+    it "saves binary one by one" do
+      @fakes.each do |name, biography|
+        zipped_biography = Zlib::Deflate.deflate biography
+        upsert = Upsert.new connection, :pets
+        assert_creates(Pet, [{:name => name, :zipped_biography => zipped_biography}]) do
+          upsert.row({:name => name}, {:zipped_biography => Upsert.binary(zipped_biography)})
+        end
+        Zlib::Inflate.inflate(Pet.find_by_name(name).zipped_biography).must_equal biography
+      end
+    end
+  end
+end

data/test/shared/correctness.rb

@@ -0,0 +1,48 @@
+shared_examples_for 'is just as correct as other ways' do
+  describe :correctness do
+    describe 'compared to native ActiveRecord' do
+      it "is as correct as than new/set/save" do
+        assert_same_result lotsa_records do |records|
+          records.each do |selector, document|
+            if pet = Pet.where(selector).first
+              pet.update_attributes document, :without_protection => true
+            else
+              pet = Pet.new
+              selector.each do |k, v|
+                pet.send "#{k}=", v
+              end
+              document.each do |k, v|
+                pet.send "#{k}=", v
+              end
+              pet.save!
+            end
+          end
+        end
+      end
+      it "is as correct as than find_or_create + update_attributes" do
+        assert_same_result lotsa_records do |records|
+          dynamic_method = nil
+          records.each do |selector, document|
+            dynamic_method ||= "find_or_create_by_#{selector.keys.join('_or_')}"
+            pet = Pet.send(dynamic_method, *selector.values)
+            pet.update_attributes document, :without_protection => true
+          end
+        end
+      end
+      it "is as correct as than create + rescue/find/update" do
+        assert_same_result lotsa_records do |records|
+          dynamic_method = nil
+          records.each do |selector, document|
+            dynamic_method ||= "find_or_create_by_#{selector.keys.join('_or_')}"
+            begin
+              Pet.create selector.merge(document), :without_protection => true
+            rescue
+              pet = Pet.send(dynamic_method, *selector.values)
+              pet.update_attributes document, :without_protection => true
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/test/{shared_examples.rb → shared/database.rb}

@@ -1,4 +1,4 @@
-shared_examples_for :database do
+shared_examples_for 'is a database with an upsert trick' do
   describe :row do
     it "works for a single row (base case)" do
       upsert = Upsert.new connection, :pets
@@ -52,26 +52,26 @@ shared_examples_for :database do
     it "works for multiple rows (base case)" do
       upsert = Upsert.new connection, :pets
       assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
-        upsert.multi do
-          row({:name => 'Jerry'}, :gender => 'male')
+        upsert.multi do |xxx|
+          xxx.row({:name => 'Jerry'}, :gender => 'male')
         end
       end
     end
     it "works for multiple rows (not changing anything)" do
       upsert = Upsert.new connection, :pets
       assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
-        upsert.multi do
-          row({:name => 'Jerry'}, :gender => 'male')
-          row({:name => 'Jerry'}, :gender => 'male')
+        upsert.multi do |xxx|
+          xxx.row({:name => 'Jerry'}, :gender => 'male')
+          xxx.row({:name => 'Jerry'}, :gender => 'male')
         end
       end
     end
     it "works for multiple rows (changing something)" do
       upsert = Upsert.new connection, :pets
       assert_creates(Pet, [{:name => 'Jerry', :gender => 'neutered'}]) do
-        upsert.multi do
-          row({:name => 'Jerry'}, :gender => 'male')
-          row({:name => 'Jerry'}, :gender => 'neutered')
+        upsert.multi do |xxx|
+          xxx.row({:name => 'Jerry'}, :gender => 'male')
+          xxx.row({:name => 'Jerry'}, :gender => 'neutered')
         end
       end
       Pet.where(:gender => 'male').count.must_equal 0