upsert 0.2.1 → 0.2.2

data/CHANGELOG

@@ -1,3 +1,9 @@
+0.2.2 / 2012-06-21
+
+* Bug fixes
+
+  * Correct and simplify how sql length is calculated when batching (streaming) MySQL upserts.
+
 0.2.1 / 2012-06-21
 
 * Enhancements

data/README.md

@@ -33,7 +33,7 @@ For bulk upserts, you probably still want to use `Upsert.stream`.
     Pet.upsert({:name => 'Jerry'}, :breed => 'beagle')
     Pet.upsert({:name => 'Pierre'}, :breed => 'tabby')
 
-### Gotchas
+### The "fixed column set" gotcha
 
 Currently, the first row you pass in determines the columns that will be used. That's useful for mass importing of many rows with the same columns, but is surprising if you're trying to use a single `Upsert` object to add arbitrary data. For example, this won't work:
 
@@ -47,7 +47,14 @@ You would need to use a new `Upsert` object. On the other hand, this is totally
     Pet.upsert({:name => 'Jerry'}, :breed => 'beagle')
     Pet.upsert({:tag_number => 456}, :spiel => 'great cat')
 
-Please send in a pull request if you think there's a better way!
+## Wishlist
+
+Pull requests for any of these would be greatly appreciated:
+
+1. Somebody who understands statistics should look at how I'm sampling rows in `Upsert::Mysql2_Client#estimate_variable_sql_bytesize`... I think we can assume that row sizes are random, so I don't think we actually have to select random elements.
+2. Fix SQLite tests.
+3. If you think there's a fix for the "fixed column set" gotcha...
+4. Naming suggestions: should "document" be called "setters" or "attributes"? Should "stream" be "batch" instead?
 
 ## Real-world usage
 

data/lib/upsert.rb

@@ -49,6 +49,7 @@ class Upsert
   USEC_SPRINTF = '%06d'
   ISO8601_DATETIME = '%Y-%m-%d %H:%M:%S'
   ISO8601_DATE = '%F'
+  NULL_WORD = 'NULL'
 
   # @return [Mysql2::Client,Sqlite3::Database,PG::Connection,#raw_connection]
   attr_reader :connection
@@ -120,7 +121,7 @@ class Upsert
   def quote_value(v)
     case v
     when NilClass
-      'NULL'
+      NULL_WORD
     when Upsert::Binary
       quote_binary v # must be defined by base
     when String
@@ -141,19 +142,4 @@ class Upsert
       raise "not sure how to quote #{v.class}: #{v.inspect}"
     end
   end
-
-  # @private
-  def quote_idents(idents)
-    idents.map { |k| quote_ident(k) }.join(',') # must be defined by base
-  end
-
-  # @private
-  def quote_values(values)
-    values.map { |v| quote_value(v) }.join(',')
-  end
-  
-  # @private
-  def quote_pairs(pairs)
-    pairs.map { |k, v| [quote_ident(k),quote_value(v)].join('=') }.join(',')
-  end
 end

data/lib/upsert/mysql2_client.rb

@@ -1,6 +1,8 @@
 class Upsert
   # @private
   module Mysql2_Client
+    SAMPLE = 0.1
+
     def chunk
       return if rows.empty?
       all = rows.length
@@ -11,9 +13,9 @@ class Upsert
       if async? and take == all
         return
       end
-      while take > 1 and oversize?(take)
+      while take > 2 and oversize?(take)
         $stderr.puts "   Length prediction via sampling failed, shrinking" if ENV['UPSERT_DEBUG'] == 'true'
-        take -= 1
+        take -= 2
       end
       chunk = sql take
       while take > 1 and chunk.bytesize > max_sql_bytesize
@@ -46,14 +48,13 @@ class Upsert
     end
 
     def insert_part
-      @insert_part ||= %{INSERT INTO "#{table_name}" (#{quote_idents(columns)}) VALUES }
+      @insert_part ||= %{INSERT INTO "#{table_name}" (#{columns.join(',')}) VALUES }
     end
 
     def update_part
       @update_part ||= begin
         updaters = columns.map do |k|
-          qk = quote_ident k
-          [ qk, "VALUES(#{qk})" ].join('=')
+          [ k, "VALUES(#{k})" ].join('=')
         end.join(',')
         %{ ON DUPLICATE KEY UPDATE #{updaters}}
       end
@@ -64,13 +65,18 @@ class Upsert
       @static_sql_bytesize ||= insert_part.bytesize + update_part.bytesize + 2
     end
 
-    # where 3 is parens and comma
+    
     def variable_sql_bytesize(take)
-      rows.first(take).inject(0) { |sum, row| sum + row.values_sql_bytesize + 3 }
+      memo = rows.first(take).inject(0) { |sum, row| sum + row.values_sql_bytesize }
+      if take > 0
+        # parens and comma
+        memo += 3*(take-1)
+      end
+      memo
     end
 
     def estimate_variable_sql_bytesize(take)
-      n = (take / 10.0).ceil
+      n = (take * SAMPLE).ceil
       sample = if RUBY_VERSION >= '1.9'
         rows.first(take).sample(n)
       else
@@ -82,7 +88,12 @@ class Upsert
         end
         memo.first(n)
       end
-      10.0 * sample.inject(0) { |sum, row| sum + row.values_sql_bytesize + 3 }
+      memo = sample.inject(0) { |sum, row| sum + row.values_sql_bytesize } / SAMPLE
+      if take > 0
+        # parens and comma
+        memo += 3*(take-1)
+      end
+      memo
     end
 
     def sql_bytesize(take)
@@ -112,33 +123,6 @@ class Upsert
       end
     end
 
-    def quoted_value_bytesize(v)
-      case v
-      when NilClass
-        4
-      when TrueClass
-        4
-      when FalseClass
-        5
-      when BigDecimal
-        v.to_s('F').bytesize
-      when Upsert::Binary
-        v.bytesize * 2 + 3
-      when Numeric
-        v.to_s.bytesize
-      when String
-        v.bytesize + 2
-      when Symbol
-        v.to_s.bytesize + 2
-      when Time, DateTime
-        24 + 2
-      when Date
-        10 + 2
-      else
-        raise "not sure how to get quoted length of #{v.class}: #{v.inspect}"
-      end
-    end
-
     def quote_boolean(v)
       v ? 'TRUE' : 'FALSE'
     end

data/lib/upsert/pg_connection.rb

@@ -14,9 +14,9 @@ class Upsert
       end
       hsh = row.to_hash
       ordered_args = column_definitions.map do |c|
-        hsh[c.name]
+        hsh[c.name] || NULL_WORD
       end
-      %{SELECT #{merge_function}(#{quote_values(ordered_args)})}
+      %{SELECT #{merge_function}(#{ordered_args.join(',')})}
     end
 
     def execute(sql)
@@ -56,12 +56,12 @@ class Upsert
     def create_merge_function(example_row)
       @merge_function = "pg_temp.merge_#{table_name}_#{Kernel.rand(1e11)}"
       execute <<-EOS
-CREATE FUNCTION #{merge_function}(#{column_definitions.map { |c| "#{quote_ident(c.input_name)} #{c.sql_type} DEFAULT #{c.default || 'NULL'}" }.join(',') }) RETURNS VOID AS
+CREATE FUNCTION #{merge_function}(#{column_definitions.map { |c| "#{c.input_name} #{c.sql_type} DEFAULT #{c.default || 'NULL'}" }.join(',') }) RETURNS VOID AS
 $$
 BEGIN
   LOOP
       -- first try to update the key
-      UPDATE #{table_name} SET #{column_definitions.map { |c| "#{quote_ident(c.name)} = #{quote_ident(c.input_name)}" }.join(',')} WHERE #{example_row.selector.keys.map { |k| "#{quote_ident(k)} = #{quote_ident([k,'input'].join('_'))}" }.join(' AND ') };
+      UPDATE #{table_name} SET #{column_definitions.map { |c| "#{c.name} = #{c.input_name}" }.join(',')} WHERE #{example_row.raw_selector.keys.map { |k| "#{quote_ident(k)} = #{quote_ident([k,'input'].join('_'))}" }.join(' AND ') };
       IF found THEN
           RETURN;
       END IF;
@@ -69,7 +69,7 @@ BEGIN
       -- if someone else inserts the same key concurrently,
       -- we could get a unique-key failure
       BEGIN
-          INSERT INTO #{table_name}(#{column_definitions.map { |c| quote_ident(c.name) }.join(',')}) VALUES (#{column_definitions.map { |c| quote_ident(c.input_name) }.join(',')});
+          INSERT INTO #{table_name}(#{column_definitions.map { |c| c.name }.join(',')}) VALUES (#{column_definitions.map { |c| c.input_name }.join(',')});
           RETURN;
       EXCEPTION WHEN unique_violation THEN
           -- Do nothing, and loop to try the UPDATE again.

data/lib/upsert/pg_connection/column_definition.rb

@@ -39,7 +39,7 @@ EOS
           res.reject do |row|
             row['name'] == auto_increment_primary_key
           end.map do |row|
-            new row['name'], row['sql_type'], row['default']
+            new connection, row['name'], row['sql_type'], row['default']
           end
         end
       end
@@ -49,9 +49,9 @@ EOS
       attr_reader :sql_type
       attr_reader :default
       
-      def initialize(name, sql_type, default)
-        @name = name
-        @input_name = "#{name}_input"
+      def initialize(connection, raw_name, sql_type, default)
+        @name = connection.quote_ident raw_name
+        @input_name = connection.quote_ident "#{raw_name}_input"
         @sql_type = sql_type
         @default = default
       end

data/lib/upsert/row.rb

@@ -2,13 +2,21 @@ class Upsert
   # @private
   class Row
     attr_reader :parent
+    attr_reader :raw_selector
     attr_reader :selector
     attr_reader :document
 
-    def initialize(parent, selector, document)
+    def initialize(parent, raw_selector, raw_document)
       @parent = parent
-      @selector = selector
-      @document = document
+      @raw_selector = raw_selector
+      @selector = raw_selector.inject({}) do |memo, (k, v)|
+        memo[parent.quote_ident(k)] = parent.quote_value(v)
+        memo
+      end
+      @document = raw_document.inject({}) do |memo, (k, v)|
+        memo[parent.quote_ident(k)] = parent.quote_value(v)
+        memo
+      end
     end
 
     def columns
@@ -16,40 +24,40 @@ class Upsert
     end
 
     def values_sql_bytesize
-      @values_sql_bytesize ||= pairs.inject(0) { |sum, (_, v)| sum + parent.quoted_value_bytesize(v) }
+      @values_sql_bytesize ||= pairs.inject(0) { |sum, (_, v)| sum + v.to_s.bytesize } + columns.length - 1
     end
 
     def values_sql
-      parent.quote_values pairs.map { |_, v| v }
+      pairs.map { |_, v| v }.join(',')
     end
 
     def columns_sql
-      parent.quote_idents columns
+      pairs.map { |k, _| k }.join(',')
     end
 
     def where_sql
-      parent.quote_pairs selector
+      selector.map { |k, v| [k, v].join('=') }.join(',')
     end
 
     def set_sql
-      parent.quote_pairs pairs
+      pairs.map { |k, v| [k, v].join('=') }.join(',')
     end
 
     def pairs
       @pairs ||= columns.map do |k|
-        value = if document.has_key?(k)
+        v = if document.has_key?(k)
           # prefer the document so that you can change rows
           document[k]
         else
           selector[k]
         end
-        [ k, value ]
+        [ k, v ]
       end
     end
 
     def to_hash
       @to_hash ||= pairs.inject({}) do |memo, (k, v)|
-        memo[k.to_s] = v
+        memo[k] = v
         memo
       end
     end

data/lib/upsert/version.rb

@@ -1,3 +1,3 @@
 class Upsert
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

data/test/test_mysql2.rb

@@ -4,7 +4,7 @@ require 'mysql2'
 system %{ mysql -u root -ppassword -e "DROP DATABASE IF EXISTS test_upsert; CREATE DATABASE test_upsert CHARSET utf8" }
 ActiveRecord::Base.establish_connection :adapter => 'mysql2', :username => 'root', :password => 'password', :database => 'test_upsert'
 
-describe "upserting on mysql2" do
+describe Upsert::Mysql2_Client do
   before do
     @opened_connections = []
     ActiveRecord::Base.connection.drop_table(Pet.table_name) rescue nil
@@ -38,4 +38,69 @@ describe "upserting on mysql2" do
   it_also "doesn't mess with timezones"
 
   it_also "doesn't blow up on reserved words"
+
+  describe '#sql_bytesize' do
+    def assert_exact(selector_proc, document_proc, show = false)
+      upsert = Upsert.new connection, :pets
+      0.upto(256) do |i|
+        upsert.rows << Upsert::Row.new(upsert, selector_proc.call(i), document_proc.call(i))
+        i.upto(upsert.rows.length) do |take|
+          expected_sql = upsert.sql(take)
+          actual = upsert.sql_bytesize(take)
+          if show and actual != expected_sql.bytesize
+            $stderr.puts
+            $stderr.puts "Expected: #{expected_sql.bytesize}"
+            $stderr.puts "Actual: #{actual}"
+            $stderr.puts expected_sql
+          end
+          actual.must_equal expected_sql.bytesize
+        end
+      end
+    end
+    def rand_string(length)
+      # http://www.dzone.com/snippets/generate-random-string-letters
+      # Array.new(length) { (rand(122-97) + 97).chr }.join
+      if RUBY_VERSION >= '1.9'
+        Array.new(length) { rand(512).chr(Encoding::UTF_8) }.join
+      else
+        Array.new(length) { rand(512) }.pack('C*')
+      end
+    end
+    it "is exact as selector length changes" do
+      selector_proc = proc do |i|
+        { :name => rand_string(i) }
+      end
+      document_proc = proc do |i|
+        {}
+      end
+      assert_exact selector_proc, document_proc
+    end
+    it "is exact as value length changes" do
+      selector_proc = proc do |i|
+        { :name => 'Jerry' }
+      end
+      document_proc = proc do |i|
+        { :spiel => rand_string(i) }
+      end
+      assert_exact selector_proc, document_proc
+    end
+    it "is exact as both selector and value length change" do
+      selector_proc = proc do |i|
+        { :name => rand_string(i) }
+      end
+      document_proc = proc do |i|
+        { :spiel => rand_string(i) }
+      end
+      assert_exact selector_proc, document_proc
+    end
+    it "is exact with numbers too" do
+      selector_proc = proc do |i|
+        { :tag_number => rand(1e5) }
+      end
+      document_proc = proc do |i|
+        { :lovability => rand }
+      end
+      assert_exact selector_proc, document_proc
+    end
+  end
 end

data/test/test_pg.rb

@@ -5,7 +5,7 @@ system %{ dropdb test_upsert }
 system %{ createdb test_upsert }
 ActiveRecord::Base.establish_connection :adapter => 'postgresql', :database => 'test_upsert'
 
-describe "upserting on postgresql" do
+describe Upsert::PG_Connection do
   before do
     @opened_connections = []
     ActiveRecord::Base.connection.drop_table(Pet.table_name) rescue nil

data/test/test_sqlite.rb

@@ -6,7 +6,7 @@ FileUtils.mkdir_p File.dirname(db_path)
 FileUtils.rm_f db_path
 ActiveRecord::Base.establish_connection :adapter => 'sqlite3', :database => db_path
 
-describe "upserting on sqlite" do
+describe Upsert::SQLite3_Database do
   before do
     @opened_connections = []
     ActiveRecord::Base.connection.drop_table(Pet.table_name) rescue nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: upsert
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
   prerelease: 
 platform: ruby
 authors: