upsert 0.1.0 → 0.1.1

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012 Seamus Abshere
+Copyright (c) 2012 Brighter Planet, Inc.
 
 MIT License
 

data/README.md

@@ -1,59 +1,89 @@
 # Upsert
 
-Finally, all those SQL MERGE tricks codified.
+Finally, all those SQL MERGE tricks codified so that you can do "upsert" on MySQL, PostgreSQL, and Sqlite.
 
 ## Usage
 
+Let's say you have...
+
+    class Pet < ActiveRecord::Base
+      # col :name
+      # col :breed
+    end
+
 ### One at a time
 
     upsert = Upsert.new Pet.connection, Pet.table_name
-    upsert.row({:name => 'Jerry'}, :breed => 'beagle')
-    upsert.row({:name => 'Pierre'}, :breed => 'tabby')
+    selector = {:name => 'Jerry'}
+    document = {:breed => 'beagle'}
+    upsert.row selector, document
 
-### Multiple upserts at once
+### Multiple upserts bundled together for speed
 
-    Upsert.new(Pet.connection, Pet.table_name).multi do |upsert|
+    Upsert.stream(Pet.connection, Pet.table_name) do |upsert|
+      # [...]
       upsert.row({:name => 'Jerry'}, :breed => 'beagle')
+      # [...]
       upsert.row({:name => 'Pierre'}, :breed => 'tabby')
+      # [...]
     end
 
-## Wishlist
+Rows are buffered in memory until it's efficient to send them to the database.
+
+## Real-world usage
 
-1. Make `c=c+1` stuff possible with `Upsert.sql('c=c+1')` or something
+<p><a href="http://brighterplanet.com"><img src="https://s3.amazonaws.com/static.brighterplanet.com/assets/logos/flush-left/inline/green/rasterized/brighter_planet-160-transparent.png" alt="Brighter Planet logo"/></a></p>
 
-## Speed
+We use `upsert` for [big data processing at Brighter Planet](http://brighterplanet.com/research) and in production at
+
+* [Brighter Planet's impact estimate web service](http://impact.brighterplanet.com)
+* [Brighter Planet's reference data web service](http://data.brighterplanet.com)
+
+Originally written to speed up the [`data_miner`](https://github.com/seamusabshere/data_miner) data mining library.
+
+## Supported databases
 
 ### MySQL
 
-(from the tests)
+Using the [mysql2](https://rubygems.org/gems/mysql2) driver.
+
+    Upsert.new Mysql2::Connection.new([...]), :pets
+
+#### Speed
+
+From the tests:
 
-    Upsert was 47% faster than faking upserts with activerecord-import
     Upsert was 77% faster than find + new/set/save
     Upsert was 84% faster than create + rescue/find/update
     Upsert was 82% faster than find_or_create + update_attributes
+    Upsert was 47% faster than faking upserts with activerecord-import
 
-### PostgreSQL
+#### SQL MERGE trick
 
-    Upsert was 73% faster than find + new/set/save
-    Upsert was 84% faster than find_or_create + update_attributes
-    Upsert was 87% faster than create + rescue/find/update
+"ON DUPLICATE KEY UPDATE" where we just set everything to the value of the insert.
 
-## Supported database drivers
+    # http://dev.mysql.com/doc/refman/5.0/en/insert-on-duplicate.html
+    INSERT INTO table (a,b,c) VALUES (1,2,3), (4,5,6)
+      ON DUPLICATE KEY UPDATE a=VALUES(a),b=VALUES(b),c=VALUES(c);
 
-1. [mysql2](https://rubygems.org/gems/mysql2) (e.g. `Upsert.new(Mysql2::Connection.new([...]), :pets)`)
-2. [sqlite3](https://rubygems.org/gems/sqlite3)
-3. [pg](https://rubygems.org/gems/pg)
-4. Any of these wrapped in an ActiveRecord connection adapter (e.g. `Upsert.new(Pet.connection, Pet.table_name)`)
+Since this is an upsert helper library, not a general-use ON DUPLICATE KEY UPDATE wrapper, you **can't** do things like `c=c+1`.
 
-## SQL merge tricks in use
+### PostgreSQL
 
-### MySQL
+Using the [pg](https://rubygems.org/gems/pg) driver.
 
-    # http://dev.mysql.com/doc/refman/5.0/en/insert-on-duplicate.html
-    INSERT INTO table (a,b,c) VALUES (1,2,3)
-      ON DUPLICATE KEY UPDATE c=c+1;
+    Upsert.new PG.connect([...]), :pets
 
-### PostgreSQL
+#### Speed
+
+From the tests:
+
+    Upsert was 73% faster than find + new/set/save
+    Upsert was 84% faster than find_or_create + update_attributes
+    Upsert was 87% faster than create + rescue/find/update
+    # (can't compare to activerecord-import because you can't fake it on pg)
+
+#### SQL MERGE trick
 
     # http://www.postgresql.org/docs/current/interactive/plpgsql-control-structures.html#PLPGSQL-ERROR-TRAPPING
     CREATE TABLE db (a INT PRIMARY KEY, b TEXT);
@@ -82,15 +112,7 @@ Finally, all those SQL MERGE tricks codified.
     SELECT merge_db(1, 'david');
     SELECT merge_db(1, 'dennis');
 
-### Sqlite
-
-    # http://stackoverflow.com/questions/2717590/sqlite-upsert-on-duplicate-key-update
-    INSERT OR IGNORE INTO visits VALUES ($ip, 0);
-    UPDATE visits SET hits = hits + 1 WHERE ip LIKE $ip;
-
-### Unused alternatives
-
-#### PostgreSQL
+The decision was made **not** to use the following because it's not straight from the manual:
 
     # http://stackoverflow.com/questions/1109061/insert-on-duplicate-update-postgresql
     UPDATE table SET field='C', field2='Z' WHERE id=3;
@@ -98,6 +120,8 @@ Finally, all those SQL MERGE tricks codified.
       SELECT 3, 'C', 'Z'
       WHERE NOT EXISTS (SELECT 1 FROM table WHERE id=3);
 
+This was also rejected because there's something we can use in the manual:
+
     # http://stackoverflow.com/questions/5269590/why-doesnt-this-rule-prevent-duplicate-key-violations
     BEGIN;
     CREATE TEMP TABLE stage_data(key_column, data_columns...) ON COMMIT DROP;
@@ -110,4 +134,77 @@ Finally, all those SQL MERGE tricks codified.
        FROM stage_data
        WHERE NOT EXISTS (SELECT 1 FROM target_data
                          WHERE target_data.key_column = stage_data.key_column)
-    END;
+    END;
+
+### Sqlite
+
+Using the [sqlite3](https://rubygems.org/gems/sqlite3) driver.
+
+    Upsert.new SQLite3::Database.open([...]), :pets
+
+#### Speed
+
+FIXME tests are segfaulting. Pull request would be lovely.
+
+#### SQL MERGE trick
+
+    # http://stackoverflow.com/questions/2717590/sqlite-upsert-on-duplicate-key-update
+    # bad example because we're not doing on-duplicate-key update
+    INSERT OR IGNORE INTO visits VALUES (127.0.0.1, 1);
+    UPDATE visits SET visits = 1 WHERE ip LIKE 127.0.0.1;
+
+### Rails / ActiveRecord
+
+(assuming that one of the other three supported drivers is being used under the covers)
+
+    Upsert.new Pet.connection, Pet.table_name
+
+#### Speed
+
+Depends on the driver being used!
+
+#### SQL MERGE trick
+
+Depends on the driver being used!
+
+## Features
+
+### Tested to be fast and portable
+
+In addition to correctness, the library's tests check that it is
+
+1. Faster than comparable upsert techniques
+2. Compatible with supported databases
+
+### Not dependent on ActiveRecord
+
+As below, all you need is a raw database connection like a `Mysql2::Connection`, `PG::Connection` or a `SQLite3::Database`. These are equivalent:
+
+    # with activerecord
+    Upsert.new ActiveRecord::Base.connection, :pets
+    # with activerecord, prettier
+    Upsert.new Pet.connection, Pet.table_name
+    # without activerecord
+    Upsert.new Mysql2::Connection.new([...]), :pets
+
+### For a specific use case, faster and more portable than `activerecord-import`
+
+You could also use [activerecord-import](https://github.com/zdennis/activerecord-import) to upsert:
+
+    Pet.import columns, all_values, :timestamps => false, :on_duplicate_key_update => columns
+
+This, however, only works on MySQL and requires ActiveRecord&mdash;and if all you are doing is upserts, `upsert` is tested to be 40% faster. And you don't have to put all of the rows to be upserted into a single huge array - you can stream them using `Upsert.stream`.
+
+### Loosely based on mongo-ruby-driver's upsert functionality
+
+The `selector` and `document` arguments are inspired by the upsert functionality of the [mongo-ruby-driver's update method](http://api.mongodb.org/ruby/1.6.4/Mongo/Collection.html#update-instance_method).
+
+## Wishlist
+
+1. `Pet.upsert`... duh
+2. Don't need a separate buffer class... just extend an instance of Upsert with the appropriate database driver module.
+
+## Copyright
+
+Copyright 2012 Brighter Planet, Inc.
+

data/lib/upsert.rb

@@ -17,6 +17,30 @@ class Upsert
     def binary(v)
       Binary.new v
     end
+
+    # @yield [Upsert] An +Upsert+ object in streaming mode. You can call #row on it multiple times and it will try to optimize on speed.
+    #
+    # @note Buffered in memory until it's efficient to send to the server a packet.
+    #
+    # @raise [Upsert::TooBig] If any row is too big to fit inside a single packet.
+    #
+    # @return [nil]
+    #
+    # @example Many at once
+    #   Upsert.stream(Pet.connection, Pet.table_name) do |upsert|
+    #     upsert.row({:name => 'Jerry'}, :breed => 'beagle')
+    #     upsert.row({:name => 'Pierre'}, :breed => 'tabby')
+    #   end
+    def stream(connection, table_name)
+      upsert = new connection, table_name
+      upsert.buffer.async!
+      yield upsert
+      upsert.buffer.sync!
+    end
+  end
+
+  # Raised if a query would be too large to send in a single packet.
+  class TooBig < RuntimeError
   end
 
   # @private
@@ -25,13 +49,18 @@ class Upsert
   # @param [Mysql2::Client,Sqlite3::Database,PG::Connection,#raw_connection] connection A supported database connection.
   # @param [String,Symbol] table_name The name of the table into which you will be upserting.
   def initialize(connection, table_name)
-    @multi_mutex = Mutex.new
     @buffer = Buffer.for connection, table_name
   end
 
+  # Upsert a row given a selector and a document.
+  #
+  # @see http://api.mongodb.org/ruby/1.6.4/Mongo/Collection.html#update-instance_method Loosely based on the upsert functionality of the mongo-ruby-driver #update method
+  #
   # @param [Hash] selector Key-value pairs that will be used to find or create a row.
   # @param [Hash] document Key-value pairs that will be set on the row, whether it previously existed or not.
   #
+  # @raise [Upsert::TooBig] If any row is too big to fit inside a single packet.
+  #
   # @return [nil]
   #
   # @example One at a time
@@ -42,23 +71,4 @@ class Upsert
     buffer.add selector, document
     nil
   end
-
-  # @yield [Upsert] An +Upsert+ object in "async" mode. You can call #row on it multiple times and it will try to optimize on speed.
-  #
-  # @return [nil]
-  #
-  # @example Many at once
-  #   Upsert.new(Pet.connection, Pet.table_name).multi do |upsert|
-  #     upsert.row({:name => 'Jerry'}, :breed => 'beagle')
-  #     upsert.row({:name => 'Pierre'}, :breed => 'tabby')
-  #   end
-  def multi
-    @multi_mutex.synchronize do
-      buffer.async = true
-      yield self
-      buffer.async = false
-      buffer.clear
-    end
-    nil
-  end
 end

data/lib/upsert/buffer.rb

@@ -17,12 +17,11 @@ class Upsert
     E_AND_SINGLE_QUOTE = %{E'}
     X_AND_SINGLE_QUOTE = %{x'}
     USEC_SPRINTF = '%06d'
-    ISO8601_DATETIME = '%Y-%m-%d %H:%M:%S' #FIXME ignores timezones i think
+    ISO8601_DATETIME = '%Y-%m-%d %H:%M:%S'
 
     attr_reader :connection
     attr_reader :table_name
     attr_reader :rows
-    attr_writer :async
     
     def initialize(connection, table_name)
       @connection = connection
@@ -34,6 +33,15 @@ class Upsert
       !!@async
     end
 
+    def async!
+      @async = true
+    end
+
+    def sync!
+      @async = false
+      clear
+    end
+
     def add(selector, document)
       rows << Row.new(self, selector, document)
       if sql = chunk

data/lib/upsert/buffer/mysql2_client.rb

@@ -5,29 +5,43 @@ class Upsert
       include Quoter
 
       def chunk
-        return false if rows.empty?
-        take = rows.length
-        until take == 1 or fits_in_single_query?(take)
+        return if rows.empty?
+        all = rows.length
+        take = all
+        while take > 1 and probably_oversize?(take)
           take -= 1
         end
-        if async? and not maximal?(take)
-          return false
+        if async? and take == all
+          return
         end
-        sql = sql take
+        while take > 1 and oversize?(take)
+          $stderr.puts "   Length prediction via sampling failed, shrinking" if ENV['UPSERT_DEBUG'] == 'true'
+          take -= 1
+        end
+        chunk = sql take
+        while take > 1 and chunk.bytesize > max_sql_bytesize
+          $stderr.puts "   Supposedly exact bytesize guess failed, shrinking" if ENV['UPSERT_DEBUG'] == 'true'
+          take -= 1
+          chunk = sql take
+        end
+        if chunk.bytesize > max_sql_bytesize
+          raise TooBig
+        end
+        $stderr.puts "   Chunk (#{take}/#{chunk.bytesize}) was #{(chunk.bytesize / max_sql_bytesize.to_f * 100).round}% of the max" if ENV['UPSERT_DEBUG'] == 'true'
         @rows = rows.drop(take)
-        sql
+        chunk
       end
 
       def execute(sql)
         connection.query sql
       end
 
-      def fits_in_single_query?(take)
-        sql_length(take) <= max_sql_length
+      def probably_oversize?(take)
+        estimate_sql_bytesize(take) > max_sql_bytesize
       end
 
-      def maximal?(take)
-        sql_length(take) >= max_sql_length
+      def oversize?(take)
+        sql_bytesize(take) > max_sql_bytesize
       end
 
       def columns
@@ -49,17 +63,26 @@ class Upsert
       end
 
       # where 2 is the parens
-      def static_sql_length
-        @static_sql_length ||= insert_part.length + update_part.length + 2
+      def static_sql_bytesize
+        @static_sql_bytesize ||= insert_part.bytesize + update_part.bytesize + 2
       end
 
       # where 3 is parens and comma
-      def variable_sql_length(take)
-        rows.first(take).inject(0) { |sum, row| sum + row.values_sql_length + 3 }
+      def variable_sql_bytesize(take)
+        rows.first(take).inject(0) { |sum, row| sum + row.values_sql_bytesize + 3 }
       end
 
-      def sql_length(take)
-        static_sql_length + variable_sql_length(take)
+      def estimate_variable_sql_bytesize(take)
+        p = (take / 10.0).ceil
+        10.0 * rows.sample(p).inject(0) { |sum, row| sum + row.values_sql_bytesize + 3 }
+      end
+
+      def sql_bytesize(take)
+        static_sql_bytesize + variable_sql_bytesize(take)
+      end
+
+      def estimate_sql_bytesize(take)
+        static_sql_bytesize + estimate_variable_sql_bytesize(take)
       end
 
       def sql(take)
@@ -67,11 +90,11 @@ class Upsert
         [ insert_part, '(', all_value_sql.join('),('), ')', update_part ].join
       end
 
-      def max_sql_length
-        @max_sql_length ||= connection.query("SHOW VARIABLES LIKE 'max_allowed_packet'", :as => :hash).first['Value'].to_i
+      def max_sql_bytesize
+        @max_sql_bytesize ||= connection.query("SHOW VARIABLES LIKE 'max_allowed_packet'", :as => :hash).first['Value'].to_i
       end
 
-      def quoted_value_length(v)
+      def quoted_value_bytesize(v)
         case v
         when NilClass
           4
@@ -80,15 +103,13 @@ class Upsert
         when FalseClass
           5
         when BigDecimal
-          v.to_s('F').length
+          v.to_s('F').bytesize
         when Upsert::Binary
-          # conservative
-          v.length * 2 + 3
+          v.bytesize * 2 + 3
         when Numeric
-          v.to_s.length
+          v.to_s.bytesize
         when String
-          # conservative
-          v.length * 2 + 2
+          v.bytesize + 2
         when Time, DateTime
           24 + 2
         when Date
@@ -106,13 +127,14 @@ class Upsert
         SINGLE_QUOTE + connection.escape(v) + SINGLE_QUOTE
       end
 
-      # We **could** do this, but I don't think it's necessary.
-      # def quote_binary(v)
-      #   X_AND_SINGLE_QUOTE + v.unpack("H*")[0] + SINGLE_QUOTE
-      # end
+      # This doubles the size of the representation.
+      def quote_binary(v)
+        X_AND_SINGLE_QUOTE + v.unpack("H*")[0] + SINGLE_QUOTE
+      end
 
       # put raw binary straight into sql
-      alias_method :quote_binary, :quote_string
+      # might work if we could get the encoding issues fixed when joining together the values for the sql
+      # alias_method :quote_binary, :quote_string
 
       def quote_time(v)
         quote_string v.strftime(ISO8601_DATETIME)

data/lib/upsert/buffer/pg_connection.rb

@@ -9,7 +9,7 @@ class Upsert
       attr_reader :merge_function
 
       def chunk
-        return false if rows.empty?
+        return if rows.empty?
         row = rows.shift
         unless merge_function
           create_merge_function row

data/lib/upsert/buffer/pg_connection/column_definition.rb

@@ -7,20 +7,20 @@ class Upsert
         class << self
           def auto_increment_primary_key(connection, table_name)
             res = connection.exec <<-EOS
-              SELECT attr.attname, seq.relname
-              FROM pg_class      seq,
-                   pg_attribute  attr,
-                   pg_depend     dep,
-                   pg_namespace  name,
-                   pg_constraint cons
-              WHERE seq.oid           = dep.objid
-                AND seq.relkind       = 'S'
-                AND attr.attrelid     = dep.refobjid
-                AND attr.attnum       = dep.refobjsubid
-                AND attr.attrelid     = cons.conrelid
-                AND attr.attnum       = cons.conkey[1]
-                AND cons.contype      = 'p'
-                AND dep.refobjid      = '#{connection.quote_ident(table_name.to_s)}'::regclass
+SELECT attr.attname, seq.relname
+FROM pg_class      seq,
+     pg_attribute  attr,
+     pg_depend     dep,
+     pg_namespace  name,
+     pg_constraint cons
+WHERE seq.oid           = dep.objid
+  AND seq.relkind       = 'S'
+  AND attr.attrelid     = dep.refobjid
+  AND attr.attnum       = dep.refobjsubid
+  AND attr.attrelid     = cons.conrelid
+  AND attr.attnum       = cons.conkey[1]
+  AND cons.contype      = 'p'
+  AND dep.refobjid      = '#{connection.quote_ident(table_name.to_s)}'::regclass
 EOS
             if hit = res.first
               hit['attname']

data/lib/upsert/buffer/sqlite3_database.rb

@@ -5,12 +5,9 @@ class Upsert
       include Quoter
 
       def chunk
-        return false if rows.empty?
+        return if rows.empty?
         row = rows.shift
-        %{
-          INSERT OR IGNORE INTO "#{table_name}" (#{row.columns_sql}) VALUES (#{row.values_sql});
-          UPDATE "#{table_name}" SET #{row.set_sql} WHERE #{row.where_sql}
-        }
+        %{INSERT OR IGNORE INTO "#{table_name}" (#{row.columns_sql}) VALUES (#{row.values_sql});UPDATE "#{table_name}" SET #{row.set_sql} WHERE #{row.where_sql}}
       end
 
       def execute(sql)

data/lib/upsert/row.rb

@@ -15,8 +15,8 @@ class Upsert
       @columns ||= (selector.keys + document.keys).uniq
     end
 
-    def values_sql_length
-      @values_sql_length ||= pairs.inject(0) { |sum, (_, v)| sum + buffer.quoted_value_length(v) }
+    def values_sql_bytesize
+      @values_sql_bytesize ||= pairs.inject(0) { |sum, (_, v)| sum + buffer.quoted_value_bytesize(v) }
     end
 
     def values_sql

data/lib/upsert/version.rb

@@ -1,3 +1,3 @@
 class Upsert
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/test/helper.rb

@@ -1,5 +1,6 @@
 require 'rubygems'
 require 'bundler/setup'
+require 'securerandom'
 require 'zlib'
 require 'benchmark'
 require 'faker'
@@ -20,6 +21,7 @@ require 'active_record_inline_schema'
 class Pet < ActiveRecord::Base
   col :name
   col :gender
+  col :spiel
   col :good, :type => :boolean
   col :lovability, :type => :float
   col :morning_walk_time, :type => :datetime
@@ -30,6 +32,8 @@ class Pet < ActiveRecord::Base
   add_index :name, :unique => true
 end
 
+# ENV['UPSERT_DEBUG'] = 'true'
+
 require 'upsert'
 
 MiniTest::Spec.class_eval do
@@ -41,20 +45,21 @@ MiniTest::Spec.class_eval do
     @records ||= begin
       memo = []
       names = []
-      50.times do
+      333.times do
         names << Faker::Name.name
       end
-      200.times do
+      2000.times do
         selector = ActiveSupport::OrderedHash.new
         selector[:name] = names.sample(1).first
         document = {
           :lovability => BigDecimal.new(rand(1e11), 2),
           :tag_number => rand(1e8),
+          :spiel => SecureRandom.hex(rand(127)),
           :good => true,
           :birthday => Time.at(rand * Time.now.to_i).to_date,
           :morning_walk_time => Time.at(rand * Time.now.to_i),
-          :home_address => Faker::Address.street_address,
-          :zipped_biography => Upsert.binary(Zlib::Deflate.deflate(Faker::Lorem.paragraphs(10).join("\n\n"), Zlib::BEST_SPEED))
+          :home_address => SecureRandom.hex(rand(1000)),
+          :zipped_biography => Upsert.binary(Zlib::Deflate.deflate(SecureRandom.hex(rand(1000)), Zlib::BEST_SPEED))
         }
         memo << [selector, document]
       end
@@ -68,10 +73,9 @@ MiniTest::Spec.class_eval do
     
     Pet.delete_all
 
-    upsert = Upsert.new connection, :pets
-    upsert.multi do |xxx|
+    Upsert.stream(connection, :pets) do |upsert|
       records.each do |selector, document|
-        xxx.row(selector, document)
+        upsert.row(selector, document)
       end
     end
     ref2 = Pet.order(:name).all.map { |pet| pet.attributes.except('id') }
@@ -101,10 +105,9 @@ MiniTest::Spec.class_eval do
     sleep 1
 
     upsert_time = Benchmark.realtime do
-      upsert = Upsert.new connection, :pets
-      upsert.multi do |xxx|
+      Upsert.stream(connection, :pets) do |upsert|
         records.each do |selector, document|
-          xxx.row(selector, document)
+          upsert.row(selector, document)
         end
       end
     end
@@ -119,9 +122,11 @@ module MiniTest::Spec::SharedExamples
   end
 
   def it_also(desc)
-    self.instance_eval do
-      MiniTest::Spec.shared_examples[desc].call
-    end
+    self.instance_eval(&MiniTest::Spec.shared_examples[desc])# do
+    #   describe desc do
+    #     .call
+    #   end
+    # end
   end
 end
 

data/test/shared/binary.rb

@@ -1,20 +1,18 @@
 shared_examples_for 'supports binary upserts' do
-  describe 'binary' do
-    before do
-      @fakes = []
-      10.times do
-        @fakes << [Faker::Name.name, Faker::Lorem.paragraphs(10).join("\n\n")]
-      end
+  before do
+    @fakes = []
+    10.times do
+      @fakes << [Faker::Name.name, Faker::Lorem.paragraphs(10).join("\n\n")]
     end
-    it "saves binary one by one" do
-      @fakes.each do |name, biography|
-        zipped_biography = Zlib::Deflate.deflate biography
-        upsert = Upsert.new connection, :pets
-        assert_creates(Pet, [{:name => name, :zipped_biography => zipped_biography}]) do
-          upsert.row({:name => name}, {:zipped_biography => Upsert.binary(zipped_biography)})
-        end
-        Zlib::Inflate.inflate(Pet.find_by_name(name).zipped_biography).must_equal biography
+  end
+  it "saves binary one by one" do
+    @fakes.each do |name, biography|
+      zipped_biography = Zlib::Deflate.deflate biography
+      upsert = Upsert.new connection, :pets
+      assert_creates(Pet, [{:name => name, :zipped_biography => zipped_biography}]) do
+        upsert.row({:name => name}, {:zipped_biography => Upsert.binary(zipped_biography)})
       end
+      Zlib::Inflate.inflate(Pet.find_by_name(name).zipped_biography).must_equal biography
     end
   end
 end

data/test/shared/correctness.rb

@@ -1,47 +1,68 @@
 shared_examples_for 'is just as correct as other ways' do
-  describe :correctness do
-    describe 'compared to native ActiveRecord' do
-      it "is as correct as than new/set/save" do
-        assert_same_result lotsa_records do |records|
-          records.each do |selector, document|
-            if pet = Pet.where(selector).first
-              pet.update_attributes document, :without_protection => true
-            else
-              pet = Pet.new
-              selector.each do |k, v|
-                pet.send "#{k}=", v
-              end
-              document.each do |k, v|
-                pet.send "#{k}=", v
-              end
-              pet.save!
+  describe 'compared to native ActiveRecord' do
+    it "is as correct as than new/set/save" do
+      assert_same_result lotsa_records do |records|
+        records.each do |selector, document|
+          if pet = Pet.where(selector).first
+            pet.update_attributes document, :without_protection => true
+          else
+            pet = Pet.new
+            selector.each do |k, v|
+              pet.send "#{k}=", v
+            end
+            document.each do |k, v|
+              pet.send "#{k}=", v
             end
+            pet.save!
           end
         end
       end
-      it "is as correct as than find_or_create + update_attributes" do
-        assert_same_result lotsa_records do |records|
-          dynamic_method = nil
-          records.each do |selector, document|
-            dynamic_method ||= "find_or_create_by_#{selector.keys.join('_or_')}"
+    end
+    it "is as correct as than find_or_create + update_attributes" do
+      assert_same_result lotsa_records do |records|
+        dynamic_method = nil
+        records.each do |selector, document|
+          dynamic_method ||= "find_or_create_by_#{selector.keys.join('_or_')}"
+          pet = Pet.send(dynamic_method, *selector.values)
+          pet.update_attributes document, :without_protection => true
+        end
+      end
+    end
+    it "is as correct as than create + rescue/find/update" do
+      assert_same_result lotsa_records do |records|
+        dynamic_method = nil
+        records.each do |selector, document|
+          dynamic_method ||= "find_or_create_by_#{selector.keys.join('_or_')}"
+          begin
+            Pet.create selector.merge(document), :without_protection => true
+          rescue
             pet = Pet.send(dynamic_method, *selector.values)
             pet.update_attributes document, :without_protection => true
           end
         end
       end
-      it "is as correct as than create + rescue/find/update" do
-        assert_same_result lotsa_records do |records|
-          dynamic_method = nil
-          records.each do |selector, document|
-            dynamic_method ||= "find_or_create_by_#{selector.keys.join('_or_')}"
-            begin
-              Pet.create selector.merge(document), :without_protection => true
-            rescue
-              pet = Pet.send(dynamic_method, *selector.values)
-              pet.update_attributes document, :without_protection => true
+    end
+  end
+  describe 'compared to activerecord-import' do
+    it "is as correct as faking upserts with activerecord-import" do
+      unless Pet.connection.respond_to?(:sql_for_on_duplicate_key_update)
+        flunk "#{Pet.connection} does not support activerecord-import's :on_duplicate_key_update"
+      end
+      assert_same_result lotsa_records do |records|
+        columns = nil
+        all_values = []
+        records.each do |selector, document|
+          columns ||= (selector.keys + document.keys).uniq
+          all_values << columns.map do |k|
+            if document.has_key?(k)
+              # prefer the document so that you can change rows
+              document[k]
+            else
+              selector[k]
             end
           end
         end
+        Pet.import columns, all_values, :timestamps => false, :on_duplicate_key_update => columns
       end
     end
   end

data/test/shared/database.rb

@@ -6,6 +6,13 @@ shared_examples_for 'is a database with an upsert trick' do
         upsert.row({:name => 'Jerry'}, {:gender => 'male'})
       end
     end
+    it "works for complex selectors" do
+      upsert = Upsert.new connection, :pets
+      assert_creates(Pet, [{:name => 'Jerry', :gender => 'male', :tag_number => 4}]) do
+        upsert.row({:name => 'Jerry', :gender => 'male'}, {:tag_number => 1})
+        upsert.row({:name => 'Jerry', :gender => 'male'}, {:tag_number => 4})
+      end
+    end
     it "works for a single row (not changing anything)" do
       upsert = Upsert.new connection, :pets
       assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
@@ -36,59 +43,31 @@ shared_examples_for 'is a database with an upsert trick' do
         upsert.row({:name => 'Inky'}, {:gender => nil})
       end
     end
-    # it "works for a single row upserted many times" do
-    #   assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
-    #     ts = (0..5).map do
-    #       Thread.new do
-    #         upsert = Upsert.new new_connection, :pets
-    #         upsert.row({:name => 'Jerry'}, {:gender => 'male'})
-    #       end
-    #     end
-    #     ts.each { |t| t.join }
-    #   end
-    # end
   end
-  describe :multi do
+  describe :stream do
     it "works for multiple rows (base case)" do
-      upsert = Upsert.new connection, :pets
       assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
-        upsert.multi do |xxx|
-          xxx.row({:name => 'Jerry'}, :gender => 'male')
+        Upsert.stream(connection, :pets) do |upsert|
+          upsert.row({:name => 'Jerry'}, :gender => 'male')
         end
       end
     end
     it "works for multiple rows (not changing anything)" do
-      upsert = Upsert.new connection, :pets
       assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
-        upsert.multi do |xxx|
-          xxx.row({:name => 'Jerry'}, :gender => 'male')
-          xxx.row({:name => 'Jerry'}, :gender => 'male')
+        Upsert.stream(connection, :pets) do |upsert|
+          upsert.row({:name => 'Jerry'}, :gender => 'male')
+          upsert.row({:name => 'Jerry'}, :gender => 'male')
         end
       end
     end
     it "works for multiple rows (changing something)" do
-      upsert = Upsert.new connection, :pets
       assert_creates(Pet, [{:name => 'Jerry', :gender => 'neutered'}]) do
-        upsert.multi do |xxx|
-          xxx.row({:name => 'Jerry'}, :gender => 'male')
-          xxx.row({:name => 'Jerry'}, :gender => 'neutered')
+        Upsert.stream(connection, :pets) do |upsert|
+          upsert.row({:name => 'Jerry'}, :gender => 'male')
+          upsert.row({:name => 'Jerry'}, :gender => 'neutered')
         end
       end
       Pet.where(:gender => 'male').count.must_equal 0
     end
-    # it "works for multiple rows upserted many times" do
-    #   assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
-    #     ts = (0..5).map do
-    #       Thread.new do
-    #         upsert = Upsert.new new_connection, :pets
-    #         upsert.multi do
-    #           row({:name => 'Jerry'}, :gender => 'male')
-    #           row({:name => 'Jerry'}, :gender => 'male')
-    #         end
-    #       end
-    #     end
-    #     ts.each { |t| t.join }
-    #   end
-    # end
   end
 end

data/test/shared/multibyte.rb

@@ -1,26 +1,37 @@
 # -*- encoding: utf-8 -*-
 shared_examples_for "supports multibyte" do
-  describe :multibyte do
-    it "works one-by-one" do
+  it "works one-by-one" do
+    assert_creates(Pet, [{:name => 'I♥NY', :gender => 'périferôl'}]) do
       upsert = Upsert.new connection, :pets
-      assert_creates(Pet, [{:name => 'I♥NY', :gender => 'périferôl'}]) do
-        upsert.row({:name => 'I♥NY'}, {:gender => 'périferôl'})
-      end
+      upsert.row({:name => 'I♥NY'}, {:gender => 'périferôl'})
     end
-    it "works serially" do
+  end
+  it "works serially" do
+    assert_creates(Pet, [{:name => 'I♥NY', :gender => 'jÚrgen'}]) do
       upsert = Upsert.new connection, :pets
-      assert_creates(Pet, [{:name => 'I♥NY', :gender => 'jÚrgen'}]) do
+      upsert.row({:name => 'I♥NY'}, {:gender => 'périferôl'})
+      upsert.row({:name => 'I♥NY'}, {:gender => 'jÚrgen'})
+    end
+  end
+  it "works streaming" do
+    assert_creates(Pet, [{:name => 'I♥NY', :gender => 'jÚrgen'}]) do
+      Upsert.stream(connection, :pets) do |upsert|
         upsert.row({:name => 'I♥NY'}, {:gender => 'périferôl'})
         upsert.row({:name => 'I♥NY'}, {:gender => 'jÚrgen'})
       end
     end
-    it "works multi" do
-      assert_creates(Pet, [{:name => 'I♥NY', :gender => 'jÚrgen'}]) do
-        Upsert.new(connection, :pets).multi do |xxx|
-          xxx.row({:name => 'I♥NY'}, {:gender => 'périferôl'})
-          xxx.row({:name => 'I♥NY'}, {:gender => 'jÚrgen'})
+  end
+  it "won't overflow" do
+    upsert = Upsert.new connection, :pets
+    if upsert.buffer.respond_to?(:max_sql_bytesize)
+      max = upsert.buffer.send(:max_sql_bytesize)
+      ticks = max / 3 - 2
+      lambda do
+        loop do
+          upsert.row({:name => 'Jerry'}, :home_address => ("日" * ticks))
+          ticks += 1
         end
-      end
+      end.must_raise Upsert::TooBig
     end
   end
 end

data/test/shared/speed.rb

@@ -1,71 +1,69 @@
 shared_examples_for 'can be speeded up with upserting' do
-  describe :speed do
-    describe 'compared to native ActiveRecord' do
-      it "is faster than new/set/save" do
-        assert_faster_than 'find + new/set/save', lotsa_records do |records|
-          records.each do |selector, document|
-            if pet = Pet.where(selector).first
-              pet.update_attributes document, :without_protection => true
-            else
-              pet = Pet.new
-              selector.each do |k, v|
-                pet.send "#{k}=", v
-              end
-              document.each do |k, v|
-                pet.send "#{k}=", v
-              end
-              pet.save!
+  describe 'compared to native ActiveRecord' do
+    it "is faster than new/set/save" do
+      assert_faster_than 'find + new/set/save', lotsa_records do |records|
+        records.each do |selector, document|
+          if pet = Pet.where(selector).first
+            pet.update_attributes document, :without_protection => true
+          else
+            pet = Pet.new
+            selector.each do |k, v|
+              pet.send "#{k}=", v
             end
+            document.each do |k, v|
+              pet.send "#{k}=", v
+            end
+            pet.save!
           end
         end
       end
-      it "is faster than find_or_create + update_attributes" do
-        assert_faster_than 'find_or_create + update_attributes', lotsa_records do |records|
-          dynamic_method = nil
-          records.each do |selector, document|
-            dynamic_method ||= "find_or_create_by_#{selector.keys.join('_or_')}"
-            pet = Pet.send(dynamic_method, *selector.values)
-            pet.update_attributes document, :without_protection => true
-          end
+    end
+    it "is faster than find_or_create + update_attributes" do
+      assert_faster_than 'find_or_create + update_attributes', lotsa_records do |records|
+        dynamic_method = nil
+        records.each do |selector, document|
+          dynamic_method ||= "find_or_create_by_#{selector.keys.join('_or_')}"
+          pet = Pet.send(dynamic_method, *selector.values)
+          pet.update_attributes document, :without_protection => true
         end
       end
-      it "is faster than create + rescue/find/update" do
-        assert_faster_than 'create + rescue/find/update', lotsa_records do |records|
-          dynamic_method = nil
-          records.each do |selector, document|
-            dynamic_method ||= "find_or_create_by_#{selector.keys.join('_or_')}"
-            begin
-              Pet.create selector.merge(document), :without_protection => true
-            rescue
-              pet = Pet.send(dynamic_method, *selector.values)
-              pet.update_attributes document, :without_protection => true
-            end
+    end
+    it "is faster than create + rescue/find/update" do
+      assert_faster_than 'create + rescue/find/update', lotsa_records do |records|
+        dynamic_method = nil
+        records.each do |selector, document|
+          dynamic_method ||= "find_or_create_by_#{selector.keys.join('_or_')}"
+          begin
+            Pet.create selector.merge(document), :without_protection => true
+          rescue
+            pet = Pet.send(dynamic_method, *selector.values)
+            pet.update_attributes document, :without_protection => true
           end
         end
       end
     end
+  end
 
-    describe 'compared to activerecord-import' do
-      it "is faster than faking upserts with activerecord-import" do
-        unless Pet.connection.respond_to?(:sql_for_on_duplicate_key_update)
-          flunk "#{Pet.connection} does not support activerecord-import's :on_duplicate_key_update"
-        end
-        assert_faster_than 'faking upserts with activerecord-import', lotsa_records do |records|
-          columns = nil
-          all_values = []
-          records.each do |selector, document|
-            columns ||= (selector.keys + document.keys).uniq
-            all_values << columns.map do |k|
-              if document.has_key?(k)
-                # prefer the document so that you can change rows
-                document[k]
-              else
-                selector[k]
-              end
+  describe 'compared to activerecord-import' do
+    it "is faster than faking upserts with activerecord-import" do
+      unless Pet.connection.respond_to?(:sql_for_on_duplicate_key_update)
+        flunk "#{Pet.connection} does not support activerecord-import's :on_duplicate_key_update"
+      end
+      assert_faster_than 'faking upserts with activerecord-import', lotsa_records do |records|
+        columns = nil
+        all_values = []
+        records.each do |selector, document|
+          columns ||= (selector.keys + document.keys).uniq
+          all_values << columns.map do |k|
+            if document.has_key?(k)
+              # prefer the document so that you can change rows
+              document[k]
+            else
+              selector[k]
             end
           end
-          Pet.import columns, all_values, :timestamps => false, :on_duplicate_key_update => columns
         end
+        Pet.import columns, all_values, :timestamps => false, :on_duplicate_key_update => columns
       end
     end
   end

data/test/shared/threaded.rb

@@ -0,0 +1,31 @@
+shared_examples_for 'is thread-safe' do
+  it "is safe to use one-by-one" do
+    upsert = Upsert.new connection, :pets
+    assert_creates(Pet, [{:name => 'Jerry', :gender => 'neutered'}]) do
+      ts = []
+      10.times do
+        ts << Thread.new do
+          sleep 0.2
+          upsert.row({:name => 'Jerry'}, :gender => 'male')
+          upsert.row({:name => 'Jerry'}, :gender => 'neutered')
+        end
+        ts.each { |t| t.join }
+      end
+    end
+  end
+  it "is safe to use streaming" do
+    assert_creates(Pet, [{:name => 'Jerry', :gender => 'neutered'}]) do
+      Upsert.stream(connection, :pets) do |upsert|
+        ts = []
+        10.times do
+          ts << Thread.new do
+            sleep 0.2
+            upsert.row({:name => 'Jerry'}, :gender => 'male')
+            upsert.row({:name => 'Jerry'}, :gender => 'neutered')
+          end
+          ts.each { |t| t.join }
+        end
+      end
+    end
+  end
+end

data/test/shared/timezones.rb

@@ -1,27 +1,25 @@
 shared_examples_for "doesn't mess with timezones" do
-  describe :timezones do
-    before do
-      @old_default_tz = ActiveRecord::Base.default_timezone
-    end
-    after do
-      ActiveRecord::Base.default_timezone = @old_default_tz
-    end
+  before do
+    @old_default_tz = ActiveRecord::Base.default_timezone
+  end
+  after do
+    ActiveRecord::Base.default_timezone = @old_default_tz
+  end
 
-    it "deals fine with UTC" do
-      ActiveRecord::Base.default_timezone = :utc
-      time = Time.now.utc
-      upsert = Upsert.new connection, :pets
-      assert_creates(Pet, [{:name => 'Jerry', :morning_walk_time => time}]) do
-        upsert.row({:name => 'Jerry'}, {:morning_walk_time => time})
-      end
+  it "deals fine with UTC" do
+    ActiveRecord::Base.default_timezone = :utc
+    time = Time.now.utc
+    upsert = Upsert.new connection, :pets
+    assert_creates(Pet, [{:name => 'Jerry', :morning_walk_time => time}]) do
+      upsert.row({:name => 'Jerry'}, {:morning_walk_time => time})
     end
-    it "won't mess with UTC" do
-      ActiveRecord::Base.default_timezone = :local
-      time = Time.now
-      upsert = Upsert.new connection, :pets
-      assert_creates(Pet, [{:name => 'Jerry', :morning_walk_time => time}]) do
-        upsert.row({:name => 'Jerry'}, {:morning_walk_time => time})
-      end
+  end
+  it "won't mess with UTC" do
+    ActiveRecord::Base.default_timezone = :local
+    time = Time.now
+    upsert = Upsert.new connection, :pets
+    assert_creates(Pet, [{:name => 'Jerry', :morning_walk_time => time}]) do
+      upsert.row({:name => 'Jerry'}, {:morning_walk_time => time})
     end
   end
 end

data/test/test_mysql2.rb

@@ -31,7 +31,9 @@ describe "upserting on mysql2" do
 
   it_also 'supports binary upserts'
 
-  it_also "supports multibyte"
+  it_also 'supports multibyte'
+
+  it_also 'is thread-safe'
 
   it_also "doesn't mess with timezones"
 end

data/test/test_pg.rb

@@ -32,7 +32,9 @@ describe "upserting on postgresql" do
 
   it_also 'supports binary upserts'
 
-  it_also "supports multibyte"
+  it_also 'supports multibyte'
+
+  it_also 'is thread-safe'
 
   it_also "doesn't mess with timezones"
 end

data/test/test_sqlite.rb

@@ -32,7 +32,9 @@ describe "upserting on sqlite" do
 
   it_also 'can be speeded up with upserting'
 
-  it_also "supports multibyte"
+  it_also 'supports multibyte'
+
+  it_also 'is thread-safe'
 
   it_also "doesn't mess with timezones"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: upsert
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-15 00:00:00.000000000 Z
+date: 2012-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sqlite3
@@ -202,6 +202,7 @@ files:
 - test/shared/database.rb
 - test/shared/multibyte.rb
 - test/shared/speed.rb
+- test/shared/threaded.rb
 - test/shared/timezones.rb
 - test/test_active_record_connection_adapter.rb
 - test/test_mysql2.rb
@@ -240,6 +241,7 @@ test_files:
 - test/shared/database.rb
 - test/shared/multibyte.rb
 - test/shared/speed.rb
+- test/shared/threaded.rb
 - test/shared/timezones.rb
 - test/test_active_record_connection_adapter.rb
 - test/test_mysql2.rb