upsert 0.0.1 → 0.1.0

data/.yardopts

@@ -0,0 +1,2 @@
+--no-private
+--readme README.md

data/README.md

@@ -2,7 +2,50 @@
 
 Finally, all those SQL MERGE tricks codified.
 
-## Supported databases
+## Usage
+
+### One at a time
+
+    upsert = Upsert.new Pet.connection, Pet.table_name
+    upsert.row({:name => 'Jerry'}, :breed => 'beagle')
+    upsert.row({:name => 'Pierre'}, :breed => 'tabby')
+
+### Multiple upserts at once
+
+    Upsert.new(Pet.connection, Pet.table_name).multi do |upsert|
+      upsert.row({:name => 'Jerry'}, :breed => 'beagle')
+      upsert.row({:name => 'Pierre'}, :breed => 'tabby')
+    end
+
+## Wishlist
+
+1. Make `c=c+1` stuff possible with `Upsert.sql('c=c+1')` or something
+
+## Speed
+
+### MySQL
+
+(from the tests)
+
+    Upsert was 47% faster than faking upserts with activerecord-import
+    Upsert was 77% faster than find + new/set/save
+    Upsert was 84% faster than create + rescue/find/update
+    Upsert was 82% faster than find_or_create + update_attributes
+
+### PostgreSQL
+
+    Upsert was 73% faster than find + new/set/save
+    Upsert was 84% faster than find_or_create + update_attributes
+    Upsert was 87% faster than create + rescue/find/update
+
+## Supported database drivers
+
+1. [mysql2](https://rubygems.org/gems/mysql2) (e.g. `Upsert.new(Mysql2::Connection.new([...]), :pets)`)
+2. [sqlite3](https://rubygems.org/gems/sqlite3)
+3. [pg](https://rubygems.org/gems/pg)
+4. Any of these wrapped in an ActiveRecord connection adapter (e.g. `Upsert.new(Pet.connection, Pet.table_name)`)
+
+## SQL merge tricks in use
 
 ### MySQL
 
@@ -12,8 +55,6 @@ Finally, all those SQL MERGE tricks codified.
 
 ### PostgreSQL
 
-#### Used
-
     # http://www.postgresql.org/docs/current/interactive/plpgsql-control-structures.html#PLPGSQL-ERROR-TRAPPING
     CREATE TABLE db (a INT PRIMARY KEY, b TEXT);
     CREATE FUNCTION merge_db(key INT, data TEXT) RETURNS VOID AS
@@ -41,7 +82,15 @@ Finally, all those SQL MERGE tricks codified.
     SELECT merge_db(1, 'david');
     SELECT merge_db(1, 'dennis');
 
-#### Alternatives (not used)
+### Sqlite
+
+    # http://stackoverflow.com/questions/2717590/sqlite-upsert-on-duplicate-key-update
+    INSERT OR IGNORE INTO visits VALUES ($ip, 0);
+    UPDATE visits SET hits = hits + 1 WHERE ip LIKE $ip;
+
+### Unused alternatives
+
+#### PostgreSQL
 
     # http://stackoverflow.com/questions/1109061/insert-on-duplicate-update-postgresql
     UPDATE table SET field='C', field2='Z' WHERE id=3;
@@ -61,10 +110,4 @@ Finally, all those SQL MERGE tricks codified.
        FROM stage_data
        WHERE NOT EXISTS (SELECT 1 FROM target_data
                          WHERE target_data.key_column = stage_data.key_column)
-    END;
-
-### Sqlite
-
-    # http://stackoverflow.com/questions/2717590/sqlite-upsert-on-duplicate-key-update
-    INSERT OR IGNORE INTO visits VALUES ($ip, 0);
-    UPDATE visits SET hits = hits + 1 WHERE ip LIKE $ip;
+    END;

data/Rakefile

@@ -10,7 +10,7 @@ Rake::TestTask.new(:_test) do |test|
 end
 
 task :test_each_db_adapter do
-  %w{ mysql2 sqlite pg }.each do |database|
+  %w{ mysql2 sqlite pg active_record_connection_adapter }.each do |database|
     puts
     puts "#{'*'*10} Running #{database} tests"
     puts

data/lib/upsert.rb

@@ -1,4 +1,7 @@
+require 'bigdecimal'
+
 require 'upsert/version'
+require 'upsert/binary'
 require 'upsert/buffer'
 require 'upsert/quoter'
 require 'upsert/row'
@@ -7,35 +10,55 @@ require 'upsert/buffer/pg_connection'
 require 'upsert/buffer/sqlite3_database'
 
 class Upsert
-  INFINITY = 1.0/0
-  SINGLE_QUOTE = %{'}
-  DOUBLE_QUOTE = %{"}
-  BACKTICK = %{`}
+  class << self
+    # @param [String] v A string containing binary data that should be inserted/escaped as such.
+    #
+    # @return [Upsert::Binary]
+    def binary(v)
+      Binary.new v
+    end
+  end
 
+  # @private
   attr_reader :buffer
 
+  # @param [Mysql2::Client,Sqlite3::Database,PG::Connection,#raw_connection] connection A supported database connection.
+  # @param [String,Symbol] table_name The name of the table into which you will be upserting.
   def initialize(connection, table_name)
     @multi_mutex = Mutex.new
     @buffer = Buffer.for connection, table_name
   end
 
+  # @param [Hash] selector Key-value pairs that will be used to find or create a row.
+  # @param [Hash] document Key-value pairs that will be set on the row, whether it previously existed or not.
+  #
+  # @return [nil]
+  #
+  # @example One at a time
+  #   upsert = Upsert.new Pet.connection, Pet.table_name
+  #   upsert.row({:name => 'Jerry'}, :breed => 'beagle')
+  #   upsert.row({:name => 'Pierre'}, :breed => 'tabby')
   def row(selector, document)
     buffer.add selector, document
+    nil
   end
 
-  def cleanup
-    buffer.cleanup
-  end
-
-  def multi(&blk)
+  # @yield [Upsert] An +Upsert+ object in "async" mode. You can call #row on it multiple times and it will try to optimize on speed.
+  #
+  # @return [nil]
+  #
+  # @example Many at once
+  #   Upsert.new(Pet.connection, Pet.table_name).multi do |upsert|
+  #     upsert.row({:name => 'Jerry'}, :breed => 'beagle')
+  #     upsert.row({:name => 'Pierre'}, :breed => 'tabby')
+  #   end
+  def multi
     @multi_mutex.synchronize do
-      begin
-        buffer.async = true
-        instance_eval(&blk)
-        buffer.cleanup
-      ensure
-        buffer.async = nil
-      end
+      buffer.async = true
+      yield self
+      buffer.async = false
+      buffer.clear
     end
+    nil
   end
 end

data/lib/upsert/binary.rb

@@ -0,0 +1,7 @@
+class Upsert
+  # A wrapper class for binary strings so that Upsert knows to escape them as such.
+  #
+  # Create them with +Upsert.binary(x)+
+  class Binary < ::String
+  end
+end

data/lib/upsert/buffer.rb

@@ -1,11 +1,24 @@
 class Upsert
+  # @private
   class Buffer
     class << self
       def for(connection, table_name)
+        if connection.respond_to?(:raw_connection)
+          # deal with ActiveRecord::Base.connection or ActiveRecord::Base.connection_pool.checkout
+          connection = connection.raw_connection
+        end
         const_get(connection.class.name.gsub(/\W+/, '_')).new connection, table_name
       end
     end
 
+    SINGLE_QUOTE = %{'}
+    DOUBLE_QUOTE = %{"}
+    BACKTICK = %{`}
+    E_AND_SINGLE_QUOTE = %{E'}
+    X_AND_SINGLE_QUOTE = %{x'}
+    USEC_SPRINTF = '%06d'
+    ISO8601_DATETIME = '%Y-%m-%d %H:%M:%S' #FIXME ignores timezones i think
+
     attr_reader :connection
     attr_reader :table_name
     attr_reader :rows
@@ -22,7 +35,7 @@ class Upsert
     end
 
     def add(selector, document)
-      rows << Row.new(selector, document)
+      rows << Row.new(self, selector, document)
       if sql = chunk
         execute sql
       end
@@ -33,26 +46,5 @@ class Upsert
         execute sql
       end
     end
-
-    def chunk
-      return if rows.empty?
-      targets = []
-      sql = nil
-      begin
-        targets << rows.pop
-        last_sql = sql
-        sql = compose(targets)
-      end until rows.empty? or targets.length >= max_targets or sql.length > max_length
-      if sql.length > max_length
-        raise if last_sql.nil?
-        sql = last_sql
-        rows << targets.pop
-      end
-      sql
-    end
-    
-    def cleanup
-      clear
-    end
   end
 end

data/lib/upsert/buffer/mysql2_client.rb

@@ -1,16 +1,20 @@
 class Upsert
   class Buffer
+    # @private
     class Mysql2_Client < Buffer
-      def compose(targets)
-        columns = targets.first.columns
-        row_inserts = targets.map { |row| row.inserts }
-        column_tautologies = columns.map do |k|
-          [ quote_ident(k), "VALUES(#{quote_ident(k)})" ].join('=')
+      include Quoter
+
+      def chunk
+        return false if rows.empty?
+        take = rows.length
+        until take == 1 or fits_in_single_query?(take)
+          take -= 1
         end
-        sql = <<-EOS
-INSERT INTO "#{table_name}" (#{quote_idents(columns)}) VALUES (#{row_inserts.map { |row_insert| quote_values(row_insert) }.join('),(') })
-ON DUPLICATE KEY UPDATE #{column_tautologies.join(',')};
-EOS
+        if async? and not maximal?(take)
+          return false
+        end
+        sql = sql take
+        @rows = rows.drop(take)
         sql
       end
 
@@ -18,30 +22,109 @@ EOS
         connection.query sql
       end
 
-      def max_targets
-        INFINITY
+      def fits_in_single_query?(take)
+        sql_length(take) <= max_sql_length
       end
 
-      def max_length
-        @max_length ||= connection.query("SHOW VARIABLES LIKE 'max_allowed_packet'", :as => :hash).first['Value'].to_i
+      def maximal?(take)
+        sql_length(take) >= max_sql_length
       end
 
-      include Quoter
+      def columns
+        @columns ||= rows.first.columns
+      end
+
+      def insert_part
+        @insert_part ||= %{INSERT INTO "#{table_name}" (#{quote_idents(columns)}) VALUES }
+      end
+
+      def update_part
+        @update_part ||= begin
+          updaters = columns.map do |k|
+            qk = quote_ident k
+            [ qk, "VALUES(#{qk})" ].join('=')
+          end.join(',')
+          %{ ON DUPLICATE KEY UPDATE #{updaters}}
+        end
+      end
+
+      # where 2 is the parens
+      def static_sql_length
+        @static_sql_length ||= insert_part.length + update_part.length + 2
+      end
+
+      # where 3 is parens and comma
+      def variable_sql_length(take)
+        rows.first(take).inject(0) { |sum, row| sum + row.values_sql_length + 3 }
+      end
+
+      def sql_length(take)
+        static_sql_length + variable_sql_length(take)
+      end
 
-      def quote_value(v)
+      def sql(take)
+        all_value_sql = rows.first(take).map { |row| row.values_sql }
+        [ insert_part, '(', all_value_sql.join('),('), ')', update_part ].join
+      end
+
+      def max_sql_length
+        @max_sql_length ||= connection.query("SHOW VARIABLES LIKE 'max_allowed_packet'", :as => :hash).first['Value'].to_i
+      end
+
+      def quoted_value_length(v)
         case v
         when NilClass
-          'NULL'
-        when String, Symbol
-          SINGLE_QUOTE + connection.escape(v.to_s) + SINGLE_QUOTE
+          4
+        when TrueClass
+          4
+        when FalseClass
+          5
+        when BigDecimal
+          v.to_s('F').length
+        when Upsert::Binary
+          # conservative
+          v.length * 2 + 3
+        when Numeric
+          v.to_s.length
+        when String
+          # conservative
+          v.length * 2 + 2
+        when Time, DateTime
+          24 + 2
+        when Date
+          10 + 2
         else
-          v
+          raise "not sure how to get quoted length of #{v.class}: #{v.inspect}"
         end
       end
-      
+
+      def quote_boolean(v)
+        v ? 'TRUE' : 'FALSE'
+      end
+
+      def quote_string(v)
+        SINGLE_QUOTE + connection.escape(v) + SINGLE_QUOTE
+      end
+
+      # We **could** do this, but I don't think it's necessary.
+      # def quote_binary(v)
+      #   X_AND_SINGLE_QUOTE + v.unpack("H*")[0] + SINGLE_QUOTE
+      # end
+
+      # put raw binary straight into sql
+      alias_method :quote_binary, :quote_string
+
+      def quote_time(v)
+        quote_string v.strftime(ISO8601_DATETIME)
+      end
+
       def quote_ident(k)
         BACKTICK + connection.escape(k.to_s) + BACKTICK
       end
+
+      def quote_big_decimal(v)
+        v.to_s('F')
+      end
     end
   end
 end

data/lib/upsert/buffer/pg_connection.rb

@@ -2,69 +2,63 @@ require 'upsert/buffer/pg_connection/column_definition'
 
 class Upsert
   class Buffer
+    # @private
     class PG_Connection < Buffer
-      attr_reader :db_function_name
+      include Quoter
+
+      attr_reader :merge_function
 
-      def compose(targets)
-        target = targets.first
-        unless created_db_function?
-          create_db_function target
+      def chunk
+        return false if rows.empty?
+        row = rows.shift
+        unless merge_function
+          create_merge_function row
         end
-        hsh = target.to_hash
+        hsh = row.to_hash
         ordered_args = column_definitions.map do |c|
-          if hsh.has_key? c.name
-            hsh[c.name]
-          else
-            nil
-          end
+          hsh[c.name]
         end
-        %{ SELECT #{db_function_name}(#{quote_values(ordered_args)}) }
+        %{SELECT #{merge_function}(#{quote_values(ordered_args)})}
       end
 
       def execute(sql)
         connection.exec sql
       end
 
-      def max_length
-        INFINITY
+      def quote_string(v)
+        SINGLE_QUOTE + connection.escape_string(v) + SINGLE_QUOTE
       end
 
-      def max_targets
-        1
+      def quote_binary(v)
+        E_AND_SINGLE_QUOTE + connection.escape_bytea(v) + SINGLE_QUOTE
       end
 
-      include Quoter
-      
-      def quote_ident(k)
-        SINGLE_QUOTE + connection.quote_ident(k) + SINGLE_QUOTE
+      def quote_time(v)
+        quote_string [v.strftime(ISO8601_DATETIME), sprintf(USEC_SPRINTF, v.usec)].join('.')
       end
-      
-      # FIXME escape_bytea with (v, k = nil)
-      def quote_value(v)
-        case v
-        when NilClass
-          'NULL'
-        when String, Symbol
-          SINGLE_QUOTE + connection.escape_string(v.to_s) + SINGLE_QUOTE
-        else
-          v
-        end
+
+      def quote_big_decimal(v)
+        v.to_s('F')
       end
-      
+
+      def quote_boolean(v)
+        v ? 'TRUE' : 'FALSE'
+      end
+
+      def quote_ident(k)
+        DOUBLE_QUOTE + connection.quote_ident(k.to_s) + DOUBLE_QUOTE
+      end
+
       def column_definitions
         @column_definitions ||= ColumnDefinition.all(connection, table_name)
       end
       
       private
       
-      def created_db_function?
-        !!@created_db_function_query
-      end
-      
-      def create_db_function(example_row)
-        @db_function_name = "pg_temp.merge_#{table_name}_#{Kernel.rand(1e11)}"
+      def create_merge_function(example_row)
+        @merge_function = "pg_temp.merge_#{table_name}_#{Kernel.rand(1e11)}"
         execute <<-EOS
-CREATE FUNCTION #{db_function_name}(#{column_definitions.map { |c| "#{c.name}_input #{c.sql_type} DEFAULT #{c.default || 'NULL'}" }.join(',') }) RETURNS VOID AS
+CREATE FUNCTION #{merge_function}(#{column_definitions.map { |c| "#{c.name}_input #{c.sql_type} DEFAULT #{c.default || 'NULL'}" }.join(',') }) RETURNS VOID AS
 $$
 BEGIN
     LOOP
@@ -87,7 +81,6 @@ END;
 $$
 LANGUAGE plpgsql;
 EOS
-        @created_db_function_query = true
       end
     end
   end