upsert 0.1.2 → 0.2.0

data/test/shared/multibyte.rb

@@ -23,8 +23,8 @@ shared_examples_for "supports multibyte" do
   end
   it "won't overflow" do
     upsert = Upsert.new connection, :pets
-    if upsert.buffer.respond_to?(:max_sql_bytesize)
-      max = upsert.buffer.send(:max_sql_bytesize)
+    if upsert.respond_to?(:max_sql_bytesize)
+      max = upsert.send(:max_sql_bytesize)
       ticks = max / 3 - 2
       lambda do
         loop do

data/test/shared/reserved_words.rb

@@ -0,0 +1,41 @@
+shared_examples_for "doesn't blow up on reserved words" do
+  # collect and uniq reserved words
+  reserved_words = ['mysql_reserved.txt', 'pg_reserved.txt'].map do |basename|
+    File.expand_path("../../misc/#{basename}", __FILE__)
+  end.map do |path|
+    IO.readlines(path)
+  end.flatten.map(&:chomp).select(&:present?).uniq
+
+  # make lots of AR models, each of which has 10 columns named after these words
+  nasties = []
+  reserved_words.each_slice(10) do |words|
+    eval %{
+      class Nasty#{nasties.length} < ActiveRecord::Base
+      end
+    }
+    nasty = Object.const_get("Nasty#{nasties.length}")
+    nasty.class_eval do
+      self.primary_key = 'fake_primary_key'
+      col :fake_primary_key
+      words.each do |word|
+        col word
+      end
+    end
+    nasties << [ nasty, words ]
+  end
+  nasties.each do |nasty, _|
+    nasty.auto_upgrade!
+  end
+
+  nasties.each do |nasty, words|
+    it "doesn't die on reserved words #{words.join(',')}" do
+      upsert = Upsert.new connection, nasty.table_name
+      random = rand(1e3).to_s
+      selector = { :fake_primary_key => random, words.first => words.first }
+      document = words[1..-1].inject({}) { |memo, word| memo[word] = word; memo }
+      assert_creates nasty, [selector.merge(document)] do
+        upsert.row selector, document
+      end
+    end
+  end
+end

data/test/test_active_record_upsert.rb

@@ -0,0 +1,23 @@
+require 'helper'
+require 'mysql2'
+
+system %{ mysql -u root -ppassword -e "DROP DATABASE IF EXISTS test_upsert; CREATE DATABASE test_upsert CHARSET utf8" }
+ActiveRecord::Base.establish_connection :adapter => 'mysql2', :username => 'root', :password => 'password', :database => 'test_upsert'
+
+require 'upsert/active_record_upsert'
+
+describe Upsert::ActiveRecordUpsert do
+  before do
+    ActiveRecord::Base.connection.drop_table(Pet.table_name) rescue nil
+    Pet.auto_upgrade!
+  end
+
+  describe :upsert do
+    it "is easy to use" do
+      assert_creates(Pet,[{:name => 'Jerry', :good => true}]) do
+        Pet.upsert({:name => 'Jerry'}, :good => false)
+        Pet.upsert({:name => 'Jerry'}, :good => true)
+      end
+    end
+  end
+end

data/test/test_mysql2.rb

@@ -36,4 +36,6 @@ describe "upserting on mysql2" do
   it_also 'is thread-safe'
 
   it_also "doesn't mess with timezones"
+
+  it_also "doesn't blow up on reserved words"
 end

data/test/test_pg.rb

@@ -37,4 +37,6 @@ describe "upserting on postgresql" do
   it_also 'is thread-safe'
 
   it_also "doesn't mess with timezones"
+
+  it_also "doesn't blow up on reserved words"
 end

data/test/test_sqlite.rb

@@ -11,19 +11,20 @@ describe "upserting on sqlite" do
     @opened_connections = []
     ActiveRecord::Base.connection.drop_table(Pet.table_name) rescue nil
     Pet.auto_upgrade!
-    @connection = new_connection
-  end
-  after do
-    @opened_connections.each { |c| c.close }
-  end
-
-  def new_connection
-    c = SQLite3::Database.open(File.expand_path('../../tmp/test.sqlite3', __FILE__))
-    @opened_connections << c
-    c
+    # @connection = new_connection
   end
+  # after do
+  #   @opened_connections.each { |c| c.close }
+  # end
+
+  # def new_connection
+  #   c = SQLite3::Database.open(File.expand_path('../../tmp/test.sqlite3', __FILE__))
+  #   @opened_connections << c
+  #   c
+  # end
   def connection
-    @connection
+    # @connection
+    ActiveRecord::Base.connection
   end
 
   it_also 'is a database with an upsert trick'
@@ -39,4 +40,6 @@ describe "upserting on sqlite" do
   it_also "doesn't mess with timezones"
 
   it_also 'supports binary upserts'
+
+  # it_also "doesn't blow up on reserved words"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: upsert
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-19 00:00:00.000000000 Z
+date: 2012-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sqlite3
@@ -187,24 +187,28 @@ files:
 - README.md
 - Rakefile
 - lib/upsert.rb
+- lib/upsert/active_record_upsert.rb
 - lib/upsert/binary.rb
-- lib/upsert/buffer.rb
-- lib/upsert/buffer/mysql2_client.rb
-- lib/upsert/buffer/pg_connection.rb
-- lib/upsert/buffer/pg_connection/column_definition.rb
-- lib/upsert/buffer/sqlite3_database.rb
-- lib/upsert/quoter.rb
+- lib/upsert/mysql2_client.rb
+- lib/upsert/pg_connection.rb
+- lib/upsert/pg_connection/column_definition.rb
 - lib/upsert/row.rb
+- lib/upsert/sqlite3_database.rb
 - lib/upsert/version.rb
 - test/helper.rb
+- test/misc/get_postgres_reserved_words.rb
+- test/misc/mysql_reserved.txt
+- test/misc/pg_reserved.txt
 - test/shared/binary.rb
 - test/shared/correctness.rb
 - test/shared/database.rb
 - test/shared/multibyte.rb
+- test/shared/reserved_words.rb
 - test/shared/speed.rb
 - test/shared/threaded.rb
 - test/shared/timezones.rb
 - test/test_active_record_connection_adapter.rb
+- test/test_active_record_upsert.rb
 - test/test_mysql2.rb
 - test/test_pg.rb
 - test/test_sqlite.rb
@@ -236,14 +240,19 @@ summary: Upsert for MySQL, PostgreSQL, and SQLite. Finally, all those SQL MERGE
   codified.
 test_files:
 - test/helper.rb
+- test/misc/get_postgres_reserved_words.rb
+- test/misc/mysql_reserved.txt
+- test/misc/pg_reserved.txt
 - test/shared/binary.rb
 - test/shared/correctness.rb
 - test/shared/database.rb
 - test/shared/multibyte.rb
+- test/shared/reserved_words.rb
 - test/shared/speed.rb
 - test/shared/threaded.rb
 - test/shared/timezones.rb
 - test/test_active_record_connection_adapter.rb
+- test/test_active_record_upsert.rb
 - test/test_mysql2.rb
 - test/test_pg.rb
 - test/test_sqlite.rb

data/lib/upsert/buffer.rb

@@ -1,58 +0,0 @@
-class Upsert
-  # @private
-  class Buffer
-    class << self
-      def for(connection, table_name)
-        if connection.respond_to?(:raw_connection)
-          # deal with ActiveRecord::Base.connection or ActiveRecord::Base.connection_pool.checkout
-          connection = connection.raw_connection
-        end
-        const_get(connection.class.name.gsub(/\W+/, '_')).new connection, table_name
-      end
-    end
-
-    SINGLE_QUOTE = %{'}
-    DOUBLE_QUOTE = %{"}
-    BACKTICK = %{`}
-    E_AND_SINGLE_QUOTE = %{E'}
-    X_AND_SINGLE_QUOTE = %{x'}
-    USEC_SPRINTF = '%06d'
-    ISO8601_DATETIME = '%Y-%m-%d %H:%M:%S'
-
-    attr_reader :connection
-    attr_reader :table_name
-    attr_reader :rows
-    
-    def initialize(connection, table_name)
-      @connection = connection
-      @table_name = table_name
-      @rows = []
-    end
-
-    def async?
-      !!@async
-    end
-
-    def async!
-      @async = true
-    end
-
-    def sync!
-      @async = false
-      clear
-    end
-
-    def add(selector, document)
-      rows << Row.new(self, selector, document)
-      if sql = chunk
-        execute sql
-      end
-    end
-
-    def clear
-      while sql = chunk
-        execute sql
-      end
-    end
-  end
-end

data/lib/upsert/buffer/mysql2_client.rb

@@ -1,164 +0,0 @@
-class Upsert
-  class Buffer
-    # @private
-    class Mysql2_Client < Buffer
-      include Quoter
-
-      def chunk
-        return if rows.empty?
-        all = rows.length
-        take = all
-        while take > 1 and probably_oversize?(take)
-          take -= 1
-        end
-        if async? and take == all
-          return
-        end
-        while take > 1 and oversize?(take)
-          $stderr.puts "   Length prediction via sampling failed, shrinking" if ENV['UPSERT_DEBUG'] == 'true'
-          take -= 1
-        end
-        chunk = sql take
-        while take > 1 and chunk.bytesize > max_sql_bytesize
-          $stderr.puts "   Supposedly exact bytesize guess failed, shrinking" if ENV['UPSERT_DEBUG'] == 'true'
-          take -= 1
-          chunk = sql take
-        end
-        if chunk.bytesize > max_sql_bytesize
-          raise TooBig
-        end
-        $stderr.puts "   Chunk (#{take}/#{chunk.bytesize}) was #{(chunk.bytesize / max_sql_bytesize.to_f * 100).round}% of the max" if ENV['UPSERT_DEBUG'] == 'true'
-        @rows = rows.drop(take)
-        chunk
-      end
-
-      def execute(sql)
-        connection.query sql
-      end
-
-      def probably_oversize?(take)
-        estimate_sql_bytesize(take) > max_sql_bytesize
-      end
-
-      def oversize?(take)
-        sql_bytesize(take) > max_sql_bytesize
-      end
-
-      def columns
-        @columns ||= rows.first.columns
-      end
-
-      def insert_part
-        @insert_part ||= %{INSERT INTO "#{table_name}" (#{quote_idents(columns)}) VALUES }
-      end
-
-      def update_part
-        @update_part ||= begin
-          updaters = columns.map do |k|
-            qk = quote_ident k
-            [ qk, "VALUES(#{qk})" ].join('=')
-          end.join(',')
-          %{ ON DUPLICATE KEY UPDATE #{updaters}}
-        end
-      end
-
-      # where 2 is the parens
-      def static_sql_bytesize
-        @static_sql_bytesize ||= insert_part.bytesize + update_part.bytesize + 2
-      end
-
-      # where 3 is parens and comma
-      def variable_sql_bytesize(take)
-        rows.first(take).inject(0) { |sum, row| sum + row.values_sql_bytesize + 3 }
-      end
-
-      def estimate_variable_sql_bytesize(take)
-        p = (take / 10.0).ceil
-        10.0 * rows.sample(p).inject(0) { |sum, row| sum + row.values_sql_bytesize + 3 }
-      end
-
-      def sql_bytesize(take)
-        static_sql_bytesize + variable_sql_bytesize(take)
-      end
-
-      def estimate_sql_bytesize(take)
-        static_sql_bytesize + estimate_variable_sql_bytesize(take)
-      end
-
-      def sql(take)
-        all_value_sql = rows.first(take).map { |row| row.values_sql }
-        [ insert_part, '(', all_value_sql.join('),('), ')', update_part ].join
-      end
-
-      # since setting an option like :as => :hash actually persists that option to the client, don't pass any options
-      def max_sql_bytesize
-        @max_sql_bytesize ||= begin
-          case (row = connection.query("SHOW VARIABLES LIKE 'max_allowed_packet'").first)
-          when Array
-            row[1]
-          when Hash
-            row['Value']
-          else
-            raise "Don't know what to do if connection.query returns a #{row.class}"
-          end.to_i
-        end
-      end
-
-      def quoted_value_bytesize(v)
-        case v
-        when NilClass
-          4
-        when TrueClass
-          4
-        when FalseClass
-          5
-        when BigDecimal
-          v.to_s('F').bytesize
-        when Upsert::Binary
-          v.bytesize * 2 + 3
-        when Numeric
-          v.to_s.bytesize
-        when String
-          v.bytesize + 2
-        when Symbol
-          v.to_s.bytesize + 2
-        when Time, DateTime
-          24 + 2
-        when Date
-          10 + 2
-        else
-          raise "not sure how to get quoted length of #{v.class}: #{v.inspect}"
-        end
-      end
-
-      def quote_boolean(v)
-        v ? 'TRUE' : 'FALSE'
-      end
-
-      def quote_string(v)
-        SINGLE_QUOTE + connection.escape(v) + SINGLE_QUOTE
-      end
-
-      # This doubles the size of the representation.
-      def quote_binary(v)
-        X_AND_SINGLE_QUOTE + v.unpack("H*")[0] + SINGLE_QUOTE
-      end
-
-      # put raw binary straight into sql
-      # might work if we could get the encoding issues fixed when joining together the values for the sql
-      # alias_method :quote_binary, :quote_string
-
-      def quote_time(v)
-        quote_string v.strftime(ISO8601_DATETIME)
-      end
-
-      def quote_ident(k)
-        BACKTICK + connection.escape(k.to_s) + BACKTICK
-      end
-
-      def quote_big_decimal(v)
-        v.to_s('F')
-      end
-    end
-  end
-end

data/lib/upsert/buffer/pg_connection.rb

@@ -1,87 +0,0 @@
-require 'upsert/buffer/pg_connection/column_definition'
-
-class Upsert
-  class Buffer
-    # @private
-    class PG_Connection < Buffer
-      include Quoter
-
-      attr_reader :merge_function
-
-      def chunk
-        return if rows.empty?
-        row = rows.shift
-        unless merge_function
-          create_merge_function row
-        end
-        hsh = row.to_hash
-        ordered_args = column_definitions.map do |c|
-          hsh[c.name]
-        end
-        %{SELECT #{merge_function}(#{quote_values(ordered_args)})}
-      end
-
-      def execute(sql)
-        connection.exec sql
-      end
-
-      def quote_string(v)
-        SINGLE_QUOTE + connection.escape_string(v) + SINGLE_QUOTE
-      end
-
-      def quote_binary(v)
-        E_AND_SINGLE_QUOTE + connection.escape_bytea(v) + SINGLE_QUOTE
-      end
-
-      def quote_time(v)
-        quote_string [v.strftime(ISO8601_DATETIME), sprintf(USEC_SPRINTF, v.usec)].join('.')
-      end
-
-      def quote_big_decimal(v)
-        v.to_s('F')
-      end
-
-      def quote_boolean(v)
-        v ? 'TRUE' : 'FALSE'
-      end
-
-      def quote_ident(k)
-        DOUBLE_QUOTE + connection.quote_ident(k.to_s) + DOUBLE_QUOTE
-      end
-
-      def column_definitions
-        @column_definitions ||= ColumnDefinition.all(connection, table_name)
-      end
-      
-      private
-      
-      def create_merge_function(example_row)
-        @merge_function = "pg_temp.merge_#{table_name}_#{Kernel.rand(1e11)}"
-        execute <<-EOS
-CREATE FUNCTION #{merge_function}(#{column_definitions.map { |c| "#{c.name}_input #{c.sql_type} DEFAULT #{c.default || 'NULL'}" }.join(',') }) RETURNS VOID AS
-$$
-BEGIN
-    LOOP
-        -- first try to update the key
-        UPDATE #{table_name} SET #{column_definitions.map { |c| "#{c.name} = #{c.name}_input" }.join(',')} WHERE #{example_row.selector.keys.map { |k| "#{k} = #{k}_input" }.join(' AND ') };
-        IF found THEN
-            RETURN;
-        END IF;
-        -- not there, so try to insert the key
-        -- if someone else inserts the same key concurrently,
-        -- we could get a unique-key failure
-        BEGIN
-            INSERT INTO #{table_name}(#{column_definitions.map { |c| c.name }.join(',')}) VALUES (#{column_definitions.map { |c| "#{c.name}_input" }.join(',')});
-            RETURN;
-        EXCEPTION WHEN unique_violation THEN
-            -- Do nothing, and loop to try the UPDATE again.
-        END;
-    END LOOP;
-END;
-$$
-LANGUAGE plpgsql;
-EOS
-      end
-    end
-  end
-end