upsert 2.1.2 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6522186c1e008a4b2386d957052e532149fc2ea3
-  data.tar.gz: f80674428af81360e0d91c24626e86abde86ccd7
+  metadata.gz: dfb6bea0fbe024f56a107ddb4db99ef247b5ebdb
+  data.tar.gz: 6a3f979f4b136debf07b327073b2e774fdd60322
 SHA512:
-  metadata.gz: cb6e925f2c76a2bab2800a609f4ccbec4b596b1b854a13293143dda2a67be30fe82f8138ab778a958033fe9acef8fedacded0819ea367daa9b850765eaf35aef
-  data.tar.gz: eb93f41e68345199be249465046e7dcf66ea1481c33d9ed14384b8155b3e33a9dea84d86fb50bd0b56fa8205cc2f5d433450799fa2f5c45637854a596890be33
+  metadata.gz: 850b04b05fab5c9ec30b9c27253fe8b751beef9dbbc7718abdd6cfbe93d6054796361cde49543018dd3afaf1c59ffe0c8baca1f2874b58053a94726cbf444112
+  data.tar.gz: 49db75e385e5d9cb3960e9f8d8927e8fa0657932e8422fe5ee7f77bcaf77a8dfc5b844b28c8a061953a285590b0c6ed9a1e8f87afb6c7c3ab1ba0d8dbdb8e312

data/.travis.yml

@@ -1,18 +1,40 @@
+sudo: required
+dist: trusty
 language: ruby
 global:
   - USERNAME=travis
-    PASSWORD=
+  - PASSWORD=
+addons:
+  apt:
+    packages:
+      # https://github.com/travis-ci/docs-travis-ci-com/pull/743
+      - haveged
+      - mysql-server-5.6
+      - mysql-client-core-5.6
+      - mysql-client-5.6
 rvm:
-  - 2.3.0
-  - 2.2.0
-  - 2.1.0
+  - 2.3
+  - 2.2
+  - 2.1
   - 1.9.3
-  - rbx-2
+  - rbx
+  - jruby-1.7
+  - jruby-9
+matrix:
+  allow_failures:
+    - rvm: rbx
 env:
-  - DB=postgresql
-  - DB=mysql
+  - DB=postgresql PGVERSION=9.4
+  - DB=postgresql PGVERSION=9.5
+  - DB=postgresql PGVERSION=9.4 UNIQUE_CONSTRAINT=true
+  - DB=postgresql PGVERSION=9.5 UNIQUE_CONSTRAINT=true
+  - DB=mysql DB_USER=root
 before_install:
+  - if [ "$DB" = 'mysql' ]; then sudo ./travis/tune_mysql.sh; fi
+# Right now the build-script is properly installing Postgres version.  We will need this to test PG 9.6 and up, though
+#  - if [ "$DB" = 'postgresql' ]; then sudo ./travis/install_postgres.sh; fi
+  - gem update --system
   - gem update bundler
   - bundle --version
-  - gem update --system 2.1.11
   - gem --version
+script: bundle exec rake spec

data/CHANGELOG

@@ -1,3 +1,10 @@
+2.2.0 / 2017-04-14
+
+* Enhancements
+
+  * Use native "upsert" on Postgres 9.5+! (thanks to @pnomolos https://github.com/seamusabshere/upsert/pull/79)
+  * More modern CI tests
+
 2.1.2 / 2016-02-25
 
 * Enhancements

data/README.md

@@ -57,6 +57,12 @@ end
 
 Batch mode is tested to be about 80% faster on PostgreSQL, MySQL, and SQLite3 than other ways to emulate upsert (see the tests, which fail if they are not faster).
 
+### Native Postgres upsert
+
+`INSERT ... ON CONFLICT DO UPDATE` is used when Postgres 9.5+ is detected and *unique indexes are in place.*
+
+If you don't have unique indexes, it will fall back to the classic Upsert gem user-defined function, which does not require indexes.
+
 ### ActiveRecord helper method
 
 ```ruby

data/lib/upsert.rb

@@ -15,7 +15,7 @@ class Upsert
     # @return [#info,#warn,#debug]
     attr_writer :logger
     MUTEX_FOR_PERFORM = Mutex.new
-    
+
     # The current logger
     # @return [#info,#warn,#debug]
     def logger
@@ -224,7 +224,7 @@ class Upsert
   def clear_database_functions
     merge_function_class.clear connection
   end
-  
+
   def merge_function(row)
     cache_key = [row.selector.keys, row.setter.keys]
     @merge_function_cache[cache_key] ||= merge_function_class.new(self, row.selector.keys, row.setter.keys, assume_function_exists?)

data/lib/upsert/connection/Java_ComMysqlJdbc_JDBC4Connection.rb

@@ -6,9 +6,13 @@ class Upsert
     class Java_ComMysqlJdbc_JDBC4Connection < Connection
       include Jdbc
 
-      # ? backtick?
       def quote_ident(k)
-        DOUBLE_QUOTE + k.to_s.gsub(DOUBLE_QUOTE, '""') + DOUBLE_QUOTE
+        if metal.useAnsiQuotedIdentifiers
+          DOUBLE_QUOTE + k.to_s.gsub(DOUBLE_QUOTE, '""') + DOUBLE_QUOTE
+        else
+          # Escape backticks by doubling them.  Ref http://dev.mysql.com/doc/refman/5.7/en/identifiers.html
+          BACKTICK + k.to_s.gsub(BACKTICK, BACKTICK + BACKTICK) + BACKTICK
+        end
       end
 
       def bind_value(v)

data/lib/upsert/connection/Java_OrgPostgresqlJdbc4_Jdbc4Connection.rb

@@ -10,6 +10,11 @@ class Upsert
       def quote_ident(k)
         DOUBLE_QUOTE + k.to_s.gsub(DOUBLE_QUOTE, '""') + DOUBLE_QUOTE
       end
+
+      def in_transaction?
+        # https://github.com/kares/activerecord-jdbc-adapter/commit/4d6e0e0c52d12b0166810dffc9f898141a23bee6
+        ![0, 4].include?(metal.get_transaction_state)
+      end
     end
   end
 end

data/lib/upsert/connection/PG_Connection.rb

@@ -3,7 +3,7 @@ class Upsert
     # @private
     class PG_Connection < Connection
       include Postgresql
-      
+
       def execute(sql, params = nil)
         if params
           # Upsert.logger.debug { %{[upsert] #{sql} with #{params.inspect}} }
@@ -21,6 +21,10 @@ class Upsert
       def binary(v)
         { :value => v.value, :format => 1 }
       end
+
+      def in_transaction?
+        ![PG::PQTRANS_IDLE, PG::PQTRANS_UNKNOWN].include?(metal.transaction_status)
+      end
     end
   end
 end

data/lib/upsert/connection/jdbc.rb

@@ -8,7 +8,9 @@ class Upsert
         java.sql.Types::OTHER       => 'getString', # ?! i guess unicode text?
         java.sql.Types::BINARY      => 'getBlob',
         java.sql.Types::LONGVARCHAR => 'getString',
+        java.sql.Types::BIGINT      => 'getLong',
         java.sql.Types::INTEGER     => 'getInt',
+        java.sql.Types::ARRAY       => ->(r, i){ r.getArray(i).array.to_ary }
       }
       java.sql.Types.constants.each do |type_name|
         i = java.sql.Types.const_get type_name
@@ -34,6 +36,11 @@ class Upsert
           setters = self.class.const_get(:SETTER)
           statement = metal.prepareStatement sql
           params.each_with_index do |v, i|
+            if v.is_a?(Fixnum) && v > 2_147_483_647
+              statement.setLong i+1, v
+              next
+            end
+
             case v
             when Upsert::Binary
               statement.setBytes i+1, binary(v)
@@ -72,6 +79,8 @@ class Upsert
             column_name, getter = cg
             if getter == 'getNull'
               row[column_name] = nil
+            elsif getter.respond_to?(:call)
+              row[column_name] = getter.call(raw_result, i)
             else
               row[column_name] = raw_result.send(getter, i)
             end
@@ -81,6 +90,10 @@ class Upsert
         statement.close
         result
       end
+
+      def in_transaction?
+        raise "Not implemented"
+      end
     end
   end
 end

data/lib/upsert/merge_function/Java_OrgPostgresqlJdbc4_Jdbc4Connection.rb

@@ -4,36 +4,24 @@ class Upsert
   class MergeFunction
     # @private
     class Java_OrgPostgresqlJdbc4_Jdbc4Connection < MergeFunction
+      ERROR_CLASS = org.postgresql.util.PSQLException
       include Postgresql
 
-      def execute(row)
-        first_try = true
-        values = []
-        values += row.selector.values
-        values += row.setter.values
-        hstore_delete_handlers.each do |hstore_delete_handler|
-          values << row.hstore_delete_keys.fetch(hstore_delete_handler.name, [])
-        end
-        begin
-          connection.execute sql, values.map { |v| connection.bind_value v }
-        rescue org.postgresql.util.PSQLException => pg_error
-          if pg_error.message =~ /function #{name}.* does not exist/i
-            if first_try
-              Upsert.logger.info %{[upsert] Function #{name.inspect} went missing, trying to recreate}
-              first_try = false
-              create!
-              retry
-            else
-              Upsert.logger.info %{[upsert] Failed to create function #{name.inspect} for some reason}
-              raise pg_error
-            end
-          else
-            raise pg_error
-          end
+      def execute_parameterized(query, args = [])
+        query_args = []
+        query = query.gsub(/\$(\d+)/) do |str|
+          query_args << args[Regexp.last_match[1].to_i - 1]
+          "?"
         end
+        controller.connection.execute(query, query_args)
       end
 
-
+      def unique_index_on_selector?
+        return @unique_index_on_selector if defined?(@unique_index_on_selector)
+        @unique_index_on_selector = schema_query.any? do |row|
+          row["index_columns"].sort == selector_keys.sort
+        end
+      end
     end
   end
 end

data/lib/upsert/merge_function/PG_Connection.rb

@@ -4,52 +4,21 @@ class Upsert
   class MergeFunction
     # @private
     class PG_Connection < MergeFunction
+      ERROR_CLASS = PG::Error
       include Postgresql
 
-      def execute(row)
-        first_try = true
-        values = []
-        values += row.selector.values
-        values += row.setter.values
-        hstore_delete_handlers.each do |hstore_delete_handler|
-          values << row.hstore_delete_keys.fetch(hstore_delete_handler.name, [])
-        end
-        Upsert.logger.debug do
-          %{[upsert]\n\tSelector: #{row.selector.inspect}\n\tSetter: #{row.setter.inspect}}
-        end
-        begin
-          connection.execute sql, values.map { |v| connection.bind_value v }
-        rescue PG::Error => pg_error
-          if pg_error.message =~ /function #{name}.* does not exist/i
-            if first_try
-              Upsert.logger.info %{[upsert] Function #{name.inspect} went missing, trying to recreate}
-              first_try = false
-              create!
-              retry
-            else
-              Upsert.logger.info %{[upsert] Failed to create function #{name.inspect} for some reason}
-              raise pg_error
-            end
-          else
-            raise pg_error
-          end
-        end
+      def execute_parameterized(query, args = [])
+        controller.connection.execute(query, args)
       end
 
-      # strangely ? can't be used as a placeholder
-      def sql
-        @sql ||= begin
-          bind_params = []
-          i = 1
-          (selector_keys.length + setter_keys.length).times do
-            bind_params << "$#{i}"
-            i += 1
-          end
-          hstore_delete_handlers.length.times do
-            bind_params << "$#{i}::text[]"
-            i += 1
-          end
-          %{SELECT #{name}(#{bind_params.join(', ')})}
+      def unique_index_on_selector?
+        return @unique_index_on_selector if defined?(@unique_index_on_selector)
+
+        type_map = PG::TypeMapByColumn.new([PG::TextDecoder::Array.new])
+        schema_query.type_map = type_map
+
+        @unique_index_on_selector = schema_query.values.any? do |row|
+          row.first.sort == selector_keys.sort
         end
       end
     end

data/lib/upsert/merge_function/postgresql.rb

@@ -40,13 +40,165 @@ class Upsert
         end
       end
 
+      attr_reader :quoted_setter_names
+      attr_reader :quoted_selector_names
+
+      def initialize(controller, *args)
+        super
+        @quoted_setter_names = setter_keys.map { |k| connection.quote_ident k }
+        @quoted_selector_names = selector_keys.map { |k| connection.quote_ident k }
+      end
+
+      def execute(row)
+        use_pg_native? ? pg_native(row) : pg_function(row)
+      end
+
+      def pg_function(row)
+        values = []
+        values += row.selector.values
+        values += row.setter.values
+        hstore_delete_handlers.each do |hstore_delete_handler|
+          values << row.hstore_delete_keys.fetch(hstore_delete_handler.name, [])
+        end
+        Upsert.logger.debug do
+          %{[upsert]\n\tSelector: #{row.selector.inspect}\n\tSetter: #{row.setter.inspect}}
+        end
+
+        first_try = true
+        begin
+          create! if connection.in_transaction? && !function_exists?
+          execute_parameterized(sql, values.map { |v| connection.bind_value v })
+        rescue self.class::ERROR_CLASS => pg_error
+          if pg_error.message =~ /function #{name}.* does not exist/i
+            if first_try
+              Upsert.logger.info %{[upsert] Function #{name.inspect} went missing, trying to recreate}
+              first_try = false
+              create!
+              retry
+            end
+            Upsert.logger.info %{[upsert] Failed to create function #{name.inspect} for some reason}
+            raise pg_error
+          else
+            raise pg_error
+          end
+        end
+      end
+
+      def function_exists?
+        # The ::int is a hack until jruby+jdbc is happy with bigints being returned
+        @function_exists ||= controller.connection.execute("SELECT count(*)::int AS cnt FROM pg_proc WHERE lower(proname) = lower('#{name}')").first["cnt"].to_i > 0
+      end
+
+      # strangely ? can't be used as a placeholder
       def sql
         @sql ||= begin
-          bind_params = Array.new(selector_keys.length + setter_keys.length, '?') + Array.new(hstore_delete_handlers.length, '?::text[]')
+          bind_params = []
+          i = 1
+          (selector_keys.length + setter_keys.length).times do
+            bind_params << "$#{i}"
+            i += 1
+          end
+          hstore_delete_handlers.length.times do
+            bind_params << "$#{i}::text[]"
+            i += 1
+          end
           %{SELECT #{name}(#{bind_params.join(', ')})}
         end
       end
 
+      def use_pg_native?
+        server_version >= 95 && unique_index_on_selector?
+      end
+
+      def server_version
+        @server_version ||=
+          controller.connection.execute("SHOW server_version").first["server_version"].split('.')[0..1].join('').to_i
+      end
+
+      def schema_query
+        execute_parameterized(
+          %{
+            SELECT array_agg(column_name::text) AS index_columns FROM information_schema.constraint_column_usage
+              JOIN pg_catalog.pg_constraint ON constraint_name::text = conname::text
+              WHERE table_name = $1 AND conrelid = $1::regclass::oid AND contype = 'u'
+              GROUP BY table_catalog, table_name, constraint_name
+          },
+          [table_name]
+        )
+      end
+
+      def pg_native(row)
+        bind_setter_values = row.setter.values.map { |v| connection.bind_value v }
+
+        upsert_sql = %{
+          INSERT INTO #{quoted_table_name} (#{quoted_setter_names.join(',')})
+          VALUES (#{insert_bind_placeholders(row).join(', ')})
+          ON CONFLICT(#{quoted_selector_names.join(', ')})
+          DO UPDATE SET (#{quoted_setter_names.join(', ')}) = (#{conflict_bind_placeholders(row).join(', ')})
+        }
+
+        execute_parameterized(upsert_sql, bind_setter_values)
+      end
+
+      def hstore_delete_function(sql, row, column_definition)
+        parts = []
+        if row.hstore_delete_keys.key?(column_definition.name)
+          parts << "DELETE("
+        end
+        parts << sql
+        if row.hstore_delete_keys.key?(column_definition.name)
+          keys = row.hstore_delete_keys[column_definition.name].map { |k| "'#{k.to_s.gsub("'", "\\'")}'" }
+          parts << ", ARRAY[#{keys.join(', ')}])"
+        end
+
+        parts.join(" ")
+      end
+
+      def insert_bind_placeholders(row)
+        if row.hstore_delete_keys.empty?
+          @insert_bind_placeholders ||= setter_column_definitions.each_with_index.map do |column_definition, i|
+            "$#{i + 1}"
+          end
+        else
+          setter_column_definitions.each_with_index.map do |column_definition, i|
+            idx = i + 1
+            if column_definition.hstore?
+              hstore_delete_function("$#{idx}", row, column_definition)
+            else
+              "$#{idx}"
+            end
+          end
+        end
+      end
+
+      def conflict_bind_placeholders(row)
+        if row.hstore_delete_keys.empty?
+          @conflict_bind_placeholders ||= setter_column_definitions.each_with_index.map do |column_definition, i|
+            idx = i + 1
+            if column_definition.hstore?
+              "CASE WHEN #{quoted_table_name}.#{column_definition.quoted_name} IS NULL THEN $#{idx} ELSE" \
+                + " (#{quoted_table_name}.#{column_definition.quoted_name} || $#{idx})" \
+                + " END"
+            else
+              "$#{idx}"
+            end
+          end
+        else
+          setter_column_definitions.each_with_index.map do |column_definition, i|
+            idx = i + 1
+            if column_definition.hstore?
+              "CASE WHEN #{quoted_table_name}.#{column_definition.quoted_name} IS NULL THEN " \
+                + hstore_delete_function("$#{idx}", row, column_definition) \
+                + " ELSE " \
+                + hstore_delete_function("(#{quoted_table_name}.#{column_definition.quoted_name} || $#{idx})", row, column_definition) \
+                + " END"
+            else
+              "$#{idx}"
+            end
+          end
+        end
+      end
+
       class HstoreDeleteHandler
         attr_reader :merge_function
         attr_reader :column_definition

data/lib/upsert/version.rb

@@ -1,3 +1,3 @@
 class Upsert
-  VERSION = '2.1.2'
+  VERSION = '2.2.0'
 end

data/spec/active_record_upsert_spec.rb

@@ -11,6 +11,16 @@ describe Upsert do
           Pet.upsert({:name => 'Jerry'}, :good => true)
         end
       end
+
+      it "doesn't fail inside a transaction" do
+        Upsert.clear_database_functions(Pet.connection)
+        expect {
+          Pet.transaction do
+            Pet.upsert({name: 'Simba'}, good: true)
+          end
+        }.to_not raise_error
+        expect(Pet.first.name).to eq('Simba')
+      end
     end
   end
 end

data/spec/correctness_spec.rb

@@ -1,3 +1,5 @@
+# encoding: utf-8
+
 require 'spec_helper'
 describe Upsert do
   describe 'clever correctness' do
@@ -78,6 +80,18 @@ describe Upsert do
       end
     end
 
+    it "works with utf-8 data" do
+      u = Upsert.new($conn, :pets)
+      records = [
+        {:name => '你好', :home_address => '人'},
+        {:name => 'Здравствуйте', :home_address => 'человек'},
+        {:name => '😀', :home_address => '😂'},
+      ]
+      assert_creates(Pet, records) do
+        records.each { |rec| u.row(rec) }
+      end
+    end
+
     it "tells you if you request a column that doesn't exist" do
       u = Upsert.new($conn, :pets)
       lambda { u.row(:gibberish => 'ba') }.should raise_error(/invalid col/i)

data/spec/database_functions_spec.rb

@@ -2,6 +2,10 @@ require 'spec_helper'
 require 'stringio'
 describe Upsert do
   describe 'database functions' do
+    version = 'postgresql' == ENV['DB'] ? Pet.connection.select_value("SHOW server_version")[0..2].split('.').join('').to_i : 0
+    before(:each) {
+      skip "Not using DB functions" if 'postgresql' == ENV['DB'] && UNIQUE_CONSTRAINT && version >= 95
+    }
     it "does not re-use merge functions across connections" do
       begin
         io = StringIO.new
@@ -15,7 +19,7 @@ describe Upsert do
         # clear, create (#2)
         Upsert.clear_database_functions($conn_factory.new_connection)
         Upsert.new($conn_factory.new_connection, :pets).row :name => 'hello'
-        
+
         io.rewind
         hits = io.read.split("\n").grep(/Creating or replacing/)
         hits.length.should == 2
@@ -23,13 +27,13 @@ describe Upsert do
         Upsert.logger = old_logger
       end
     end
-    
+
     it "does not re-use merge functions even when on the same connection" do
       begin
         io = StringIO.new
         old_logger = Upsert.logger
         Upsert.logger = Logger.new io, Logger::INFO
-        
+
         connection = $conn_factory.new_connection
 
         # clear, create (#1)
@@ -39,7 +43,7 @@ describe Upsert do
         # clear, create (#2)
         Upsert.clear_database_functions(connection)
         Upsert.new(connection, :pets).row :name => 'hello'
-        
+
         io.rewind
         hits = io.read.split("\n").grep(/Creating or replacing/)
         hits.length.should == 2
@@ -47,7 +51,7 @@ describe Upsert do
         Upsert.logger = old_logger
       end
     end
-    
+
     it "re-uses merge functions within batch" do
       begin
         io = StringIO.new
@@ -56,13 +60,13 @@ describe Upsert do
 
         # clear
         Upsert.clear_database_functions($conn_factory.new_connection)
-        
+
         # create
         Upsert.batch($conn_factory.new_connection, :pets) do |upsert|
           upsert.row :name => 'hello'
           upsert.row :name => 'world'
         end
-        
+
         io.rewind
         hits = io.read.split("\n").grep(/Creating or replacing/)
         hits.length.should == 1
@@ -79,7 +83,7 @@ describe Upsert do
 
         # clear
         Upsert.clear_database_functions($conn_factory.new_connection)
-        
+
         # tries, "went missing", creates
         Upsert.new($conn_factory.new_connection, :pets, :assume_function_exists => true).row :name => 'hello'
 
@@ -94,6 +98,5 @@ describe Upsert do
         Upsert.logger = old_logger
       end
     end
-
   end
 end if %w{ postgresql mysql }.include?(ENV['DB'])

data/spec/hstore_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe Upsert do
   describe 'hstore on pg' do
     require 'pg_hstore'
-    Pet.connection.execute 'CREATE EXTENSION HSTORE'
+    Pet.connection.execute 'CREATE EXTENSION IF NOT EXISTS HSTORE'
     Pet.connection.execute "ALTER TABLE pets ADD COLUMN crazy HSTORE"
     Pet.connection.execute "ALTER TABLE pets ADD COLUMN cool HSTORE"
 
@@ -11,8 +11,9 @@ describe Upsert do
       Pet.delete_all
     end
 
+    let(:upsert) { Upsert.new $conn, :pets }
+
     it "works for ugly text" do
-      upsert = Upsert.new $conn, :pets
       uggy = <<-EOS
 {"results":[{"locations":[],"providedLocation":{"location":"3001 STRATTON WAY, MADISON, WI 53719 UNITED STATES"}}],"options":{"ignoreLatLngInput":true,"maxResults":1,"thumbMaps":false},"info":{"copyright":{"text":"© 2012 MapQuest, Inc.","imageUrl":"http://api.mqcdn.com/res/mqlogo.gif","imageAltText":"© 2012 MapQuest, Inc."},"statuscode":0,"messages":[]}}
 EOS
@@ -23,8 +24,6 @@ EOS
     end
 
     it "just works" do
-      upsert = Upsert.new $conn, :pets
-
       upsert.row({:name => 'Bill'}, :crazy => nil)
       row = Pet.connection.select_one(%{SELECT crazy FROM pets WHERE name = 'Bill'})
       row['crazy'].should == nil
@@ -60,8 +59,6 @@ EOS
     end
 
     it "can nullify entire hstore" do
-      upsert = Upsert.new $conn, :pets
-
       upsert.row({:name => 'Bill'}, :crazy => {:a => 1})
       row = Pet.connection.select_one(%{SELECT crazy FROM pets WHERE name = 'Bill'})
       crazy = PgHstore.parse row['crazy']
@@ -73,8 +70,6 @@ EOS
     end
 
     it "deletes keys that are nil" do
-      upsert = Upsert.new $conn, :pets
-
       upsert.row({:name => 'Bill'}, :crazy => nil)
       row = Pet.connection.select_one(%{SELECT crazy FROM pets WHERE name = 'Bill'})
       row['crazy'].should == nil
@@ -121,8 +116,6 @@ EOS
     end
 
     it "takes dangerous keys" do
-      upsert = Upsert.new $conn, :pets
-
       upsert.row({:name => 'Bill'}, :crazy => nil)
       row = Pet.connection.select_one(%{SELECT crazy FROM pets WHERE name = 'Bill'})
       row['crazy'].should == nil
@@ -169,7 +162,6 @@ EOS
     end
 
     it "handles multiple hstores" do
-      upsert = Upsert.new $conn, :pets
       upsert.row({:name => 'Bill'}, :crazy => {:a => 1, :b => 9}, :cool => {:c => 12, :d => 19})
       row = Pet.connection.select_one(%{SELECT crazy, cool FROM pets WHERE name = 'Bill'})
       crazy = PgHstore.parse row['crazy']
@@ -179,8 +171,6 @@ EOS
     end
 
     it "can deletes keys from multiple hstores at once" do
-      upsert = Upsert.new $conn, :pets
-
       upsert.row({:name => 'Bill'}, :crazy => {:a => 1}, :cool => {5 => 9})
       row = Pet.connection.select_one(%{SELECT crazy, cool FROM pets WHERE name = 'Bill'})
       crazy = PgHstore.parse row['crazy']
@@ -217,8 +207,6 @@ EOS
     end
 
     it "deletes keys whether new or existing record" do
-      upsert = Upsert.new $conn, :pets
-
       upsert.row({:name => 'Bill'}, :crazy => {:z => 1, :x => nil})
       row = Pet.connection.select_one(%{SELECT crazy FROM pets WHERE name = 'Bill'})
       crazy = PgHstore.parse row['crazy']
@@ -231,8 +219,6 @@ EOS
     end
 
     it "can turn off eager nullify" do
-      upsert = Upsert.new $conn, :pets
-
       upsert.row({:name => 'Bill'}, {:crazy => {:z => 1, :x => nil}}, :eager_nullify => false)
       row = Pet.connection.select_one(%{SELECT crazy FROM pets WHERE name = 'Bill'})
       crazy = PgHstore.parse row['crazy']

data/spec/jruby_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+describe Upsert do
+  it "works correct with large ints" do
+    u = Upsert.new($conn, :pets)
+    Pet.create(:name => "Jerry", :big_tag_number => 2)
+    u.row({ :name => 'Jerry' }, :big_tag_number => 3599657714)
+    Pet.find_by_name('Jerry').big_tag_number.should == 3599657714
+  end
+end if RUBY_PLATFORM == 'java'

data/spec/logger_spec.rb

@@ -20,12 +20,12 @@ describe Upsert do
     end
 
     it "logs queries" do
+      old_logger = Upsert.logger
       begin
-        old_logger = Upsert.logger
         io = StringIO.new
         MUTEX_FOR_PERFORM.synchronize do
           Upsert.logger = Logger.new(io)
-          
+
           u = Upsert.new($conn, :pets)
           u.row(:name => 'Jerry')
 
@@ -38,7 +38,8 @@ describe Upsert do
             log.should =~ /call #{Upsert::MergeFunction::NAME_PREFIX}_pets_SEL_name/i
           when /p.*g/i
             # [54ae2eea857] Possibly much more useful debug output
-            log.should =~ /selector:/i
+            # TODO: Should check for both upsert and non-upsert log output
+            log.should =~ /selector:|SHOW server_version/i
           else
             raise "not sure"
           end

data/spec/postgresql_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+describe Upsert do
+  version = Pet.connection.select_value("SHOW server_version")[0..2].split('.').join('').to_i
+
+  let(:upsert) do
+    Upsert.new($conn, :pets)
+  end
+
+  it "uses the native method if available (#{(UNIQUE_CONSTRAINT && version >= 95).inspect})" do
+    p = Pet.create(:name => 'Jerry', :tag_number => 5)
+    upsert.row({ :name => 'Jerry'}, :tag_number => 6 )
+    expect(upsert.instance_variable_get(:@merge_function_cache).values.first.use_pg_native?).to(
+      UNIQUE_CONSTRAINT && version >= 95 ? be_truthy : be_falsey
+    )
+  end
+end if ENV['DB'] == 'postgresql'

data/spec/reserved_words_spec.rb

@@ -7,7 +7,7 @@ describe Upsert do
     end.map do |path|
       IO.readlines(path)
     end.flatten.map(&:chomp).select(&:present?).uniq
-  
+
     # make lots of AR models, each of which has 10 columns named after these words
     nasties = []
     reserved_words.each_slice(10) do |words|
@@ -18,9 +18,9 @@ describe Upsert do
       nasty = Object.const_get("Nasty#{nasties.length}")
       nasty.class_eval do
         self.primary_key = 'fake_primary_key'
-        col :fake_primary_key
+        col :fake_primary_key, limit: 191
         words.each do |word|
-          col word
+          col word, limit: 191
         end
       end
       nasties << [ nasty, words ]
@@ -28,7 +28,7 @@ describe Upsert do
     nasties.each do |nasty, _|
       nasty.auto_upgrade!
     end
-  
+
     describe "reserved words" do
       nasties.each do |nasty, words|
         it "doesn't die on reserved words #{words.join(',')}" do
@@ -43,4 +43,4 @@ describe Upsert do
       end
     end
   end
-end
+end

data/spec/sequel_spec.rb

@@ -4,17 +4,27 @@ require 'sequel'
 describe Upsert do
   describe "Plays nice with Sequel" do
     config = ActiveRecord::Base.connection.instance_variable_get(:@config)
-    case
-      when 'postgresql' == config[:adapter]; config[:adapter] = 'postgres'
-      when 'sqlite3' == config[:adapter]; config[:adapter] = 'sqlite'
+    config[:adapter] = case config[:adapter]
+                       when 'postgresql' then 'postgres'
+                       when 'sqlie3' then 'sqlite'
+                       else config[:adapter]
+                       end
+
+    let(:db) do
+      params = if RUBY_PLATFORM == 'java'
+                 RawConnectionFactory::CONFIG
+               else
+                 config.slice(:adapter, :host, :database, :username, :password).merge(:user => config[:username])
+               end
+      Sequel.connect(params)
     end
 
     it "Doesn't explode on connection" do
-      expect { DB = Sequel.connect config }.to_not raise_error
+      expect { db }.to_not raise_error
     end
 
     it "Doesn't explode when using DB.pool.hold" do
-      DB.pool.hold do |conn|
+      db.pool.hold do |conn|
         expect {
           upsert = Upsert.new(conn, :pets)
           assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
@@ -25,7 +35,7 @@ describe Upsert do
     end
 
     it "Doesn't explode when using DB.synchronize" do
-      DB.synchronize do |conn|
+      db.synchronize do |conn|
         expect {
           upsert = Upsert.new(conn, :pets)
           assert_creates(Pet, [{:name => 'Jerry', :gender => 'male'}]) do
@@ -35,4 +45,4 @@ describe Upsert do
       end
     end
   end
-end
+end

data/spec/spec_helper.rb

@@ -1,7 +1,8 @@
 # -*- encoding: utf-8 -*-
 require 'bundler/setup'
 
-require 'pry'
+# require 'pry'
+require 'shellwords'
 
 require 'active_record'
 ActiveRecord::Base.default_timezone = :utc
@@ -11,11 +12,14 @@ require 'active_record_inline_schema'
 require 'activerecord-import' if RUBY_VERSION >= '1.9'
 
 ENV['DB'] ||= 'mysql'
+ENV['DB'] = 'postgresql' if ENV['DB'].to_s =~ /postgresql/
+UNIQUE_CONSTRAINT = ENV['UNIQUE_CONSTRAINT'] == 'true'
+
 
 class RawConnectionFactory
   DATABASE = 'upsert_test'
-  CURRENT_USER = `whoami`.chomp
-  PASSWORD = ''
+  CURRENT_USER = (ENV['DB_USER'] || `whoami`.chomp)
+  PASSWORD = ENV['DB_PASSWORD']
 
   case ENV['DB']
 
@@ -41,9 +45,9 @@ class RawConnectionFactory
     ActiveRecord::Base.establish_connection :adapter => 'postgresql', :database => DATABASE, :username => CURRENT_USER
 
   when 'mysql'
-    password_argument = (PASSWORD.empty?) ? "" : "-p#{PASSWORD}"
-    Kernel.system %{ mysql -u #{CURRENT_USER} #{password_argument} -e "DROP DATABASE IF EXISTS #{DATABASE}" }
-    Kernel.system %{ mysql -u #{CURRENT_USER} #{password_argument} -e "CREATE DATABASE #{DATABASE} CHARSET utf8" }
+    password_argument = (PASSWORD.nil?) ? "" : "--password=#{Shellwords.escape(PASSWORD)}"
+    Kernel.system %{ mysql -h 127.0.0.1 -u #{CURRENT_USER} #{password_argument} -e "DROP DATABASE IF EXISTS #{DATABASE}" }
+    Kernel.system %{ mysql -h 127.0.0.1 -u #{CURRENT_USER} #{password_argument} -e "CREATE DATABASE #{DATABASE} CHARSET utf8mb4 COLLATE utf8mb4_general_ci" }
     if RUBY_PLATFORM == 'java'
       CONFIG = "jdbc:mysql://127.0.0.1/#{DATABASE}?user=#{CURRENT_USER}&password=#{PASSWORD}"
       require 'jdbc/mysql'
@@ -55,12 +59,20 @@ class RawConnectionFactory
     else
       require 'mysql2'
       def new_connection
-        config = { :username => CURRENT_USER, :database => DATABASE, :host => "127.0.0.1" }
-        config.merge!(:password => PASSWORD) unless PASSWORD.empty?
+        config = { :username => CURRENT_USER, :database => DATABASE, :host => "127.0.0.1", :encoding => 'utf8mb4' }
+        config.merge!(:password => PASSWORD) unless PASSWORD.nil?
         Mysql2::Client.new config
       end
     end
-    ActiveRecord::Base.establish_connection "#{RUBY_PLATFORM == 'java' ? 'mysql' : 'mysql2'}://#{CURRENT_USER}:#{PASSWORD}@127.0.0.1/#{DATABASE}"
+    ActiveRecord::Base.establish_connection(
+      :adapter => RUBY_PLATFORM == 'java' ? 'mysql' : 'mysql2',
+      :user => CURRENT_USER,
+      :password => PASSWORD,
+      :host => '127.0.0.1',
+      :database => DATABASE,
+      :encoding => 'utf8mb4'
+    )
+    ActiveRecord::Base.connection.execute "SET NAMES utf8mb4 COLLATE utf8mb4_general_ci"
 
   when 'sqlite3'
     CONFIG = { :adapter => 'sqlite3', :database => 'file::memory:?cache=shared' }
@@ -102,7 +114,7 @@ else
 end
 
 class Pet < ActiveRecord::Base
-  col :name
+  col :name, limit: 191 # utf8mb4 in mysql requirement
   col :gender
   col :spiel
   col :good, :type => :boolean
@@ -110,6 +122,7 @@ class Pet < ActiveRecord::Base
   col :morning_walk_time, :type => :datetime
   col :zipped_biography, :type => :binary
   col :tag_number, :type => :integer
+  col :big_tag_number, :type => :bigint
   col :birthday, :type => :date
   col :home_address, :type => :text
   if ENV['DB'] == 'postgresql'
@@ -117,8 +130,19 @@ class Pet < ActiveRecord::Base
   end
   add_index :name, :unique => true
 end
+if ENV['DB'] == 'postgresql' && UNIQUE_CONSTRAINT
+  begin
+    Pet.connection.execute("ALTER TABLE pets DROP CONSTRAINT IF EXISTS unique_name")
+  rescue => e
+    puts e.inspect
+  end
+end
+
 Pet.auto_upgrade!
 
+if ENV['DB'] == 'postgresql' && UNIQUE_CONSTRAINT
+  Pet.connection.execute("ALTER TABLE pets ADD CONSTRAINT unique_name UNIQUE (name)")
+end
 
 class Task < ActiveRecord::Base
   col :name
@@ -183,7 +207,7 @@ module SpecHelper
   def assert_same_result(records, &blk)
     blk.call(records)
     ref1 = Pet.order(:name).all.map { |pet| pet.attributes.except('id') }
-    
+
     Pet.delete_all
 
     Upsert.batch($conn, :pets) do |upsert|
@@ -204,6 +228,7 @@ module SpecHelper
     expected_records.each do |selector, setter|
       setter ||= {}
       found = model.where(selector).map { |record| record.attributes.except('id') }
+      expect(found).to_not be_empty, { :selector => selector, :setter => setter }.inspect
       expected = [ selector.stringify_keys.merge(setter.stringify_keys) ]
       compare_attribute_sets expected, found
     end
@@ -237,7 +262,7 @@ module SpecHelper
     Pet.delete_all
     sleep 1
     # --
-    
+
     ar_time = Benchmark.realtime { blk.call(records) }
 
     Pet.delete_all

data/spec/speed_spec.rb

@@ -46,7 +46,7 @@ describe Upsert do
       end
     end
 
-    # FIXME apparently no longer faster?  
+    # FIXME apparently no longer faster?
     # if ENV['DB'] == 'mysql' && RUBY_VERSION >= '1.9'
     #   describe 'compared to activerecord-import' do
     #     it "is faster than faking upserts with activerecord-import" do

data/travis/install_postgres.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Ref https://github.com/clkao/plv8x/blob/master/.travis.yml
+echo "Install postgres version $PGVERSION"
+psql --version
+sudo /etc/init.d/postgresql stop
+sudo apt-get -y --purge remove postgresql libpq-dev libpq5 postgresql-client-common postgresql-common
+sudo rm -rf /var/lib/postgresql
+wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+sudo sh -c "echo deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main $PGVERSION >> /etc/apt/sources.list.d/postgresql.list"
+sudo sh -c "echo deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg-testing main $PGVERSION >> /etc/apt/sources.list.d/postgresql.list"
+sudo apt-get update -qq
+sudo apt-get -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::="--force-confnew" install postgresql-$PGVERSION postgresql-server-dev-$PGVERSION postgresql-$PGVERSION-plv8
+sudo chmod 777 /etc/postgresql/$PGVERSION/main/pg_hba.conf
+sudo echo "local   all         postgres                          trust" > /etc/postgresql/$PGVERSION/main/pg_hba.conf
+sudo echo "local   all         all                               trust" >> /etc/postgresql/$PGVERSION/main/pg_hba.conf
+sudo echo "host    all         all         127.0.0.1/32          trust" >> /etc/postgresql/$PGVERSION/main/pg_hba.conf
+sudo echo "host    all         all         ::1/128               trust" >> /etc/postgresql/$PGVERSION/main/pg_hba.conf
+sudo /etc/init.d/postgresql restart

data/travis/tune_mysql.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Ref https://github.com/travis-ci/travis-ci/issues/2250
+# Ref https://github.com/jruby/activerecord-jdbc-adapter/issues/481
+# JDBC adapters determine the encoding automatically via the server's charset and collation
+echo "Tuning MySQL"
+echo -e "[server]\ncharacter-set-server=utf8mb4\ncollation-server=utf8mb4_unicode_ci\n" | sudo tee -a /etc/mysql/my.cnf
+sudo service mysql restart

data/upsert.gemspec

@@ -2,7 +2,7 @@
 require File.expand_path('../lib/upsert/version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.authors       = ["Seamus Abshere"]
+  gem.authors       = ["Seamus Abshere", "Phil Schalm"]
   gem.email         = ["seamus@abshere.net"]
   t = %{Make it easy to upsert on MySQL, PostgreSQL, and SQLite3. Transparently creates merge functions for MySQL and PostgreSQL; on SQLite3, uses INSERT OR IGNORE.}
   gem.description   = t
@@ -32,7 +32,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'rake', '~>10.1.1'
 
   if RUBY_VERSION >= '1.9'
-    gem.add_development_dependency 'activerecord-import'
+    gem.add_development_dependency 'activerecord-import', '0.11.0' # 0.12 and up were failing
   else
     gem.add_development_dependency 'orderedhash'
   end
@@ -46,11 +46,10 @@ Gem::Specification.new do |gem|
     gem.add_development_dependency 'activerecord-jdbcmysql-adapter'
     gem.add_development_dependency 'activerecord-jdbcpostgresql-adapter'
   else
-    gem.add_development_dependency 'activerecord-mysql2-adapter'
     gem.add_development_dependency 'activerecord-postgresql-adapter'
     gem.add_development_dependency 'sqlite3'
-    gem.add_development_dependency 'mysql2'
-    gem.add_development_dependency 'pg'
+    gem.add_development_dependency 'mysql2', '~> 0.3.10'
+    gem.add_development_dependency 'pg', '~> 0.18.0'
     # github-flavored markdown
     if RUBY_VERSION >= '1.9'
       gem.add_development_dependency 'redcarpet'

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: upsert
 version: !ruby/object:Gem::Version
-  version: 2.1.2
+  version: 2.2.0
 platform: ruby
 authors:
 - Seamus Abshere
+- Phil Schalm
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-25 00:00:00.000000000 Z
+date: 2017-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec-core
@@ -168,30 +169,16 @@ dependencies:
   name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.11.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: activerecord-mysql2-adapter
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.11.0
 - !ruby/object:Gem::Dependency
   name: activerecord-postgresql-adapter
   requirement: !ruby/object:Gem::Requirement
@@ -224,30 +211,30 @@ dependencies:
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.3.10
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.3.10
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.18.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.18.0
 - !ruby/object:Gem::Dependency
   name: redcarpet
   requirement: !ruby/object:Gem::Requirement
@@ -313,11 +300,13 @@ files:
 - spec/database_functions_spec.rb
 - spec/database_spec.rb
 - spec/hstore_spec.rb
+- spec/jruby_spec.rb
 - spec/logger_spec.rb
 - spec/misc/get_postgres_reserved_words.rb
 - spec/misc/mysql_reserved.txt
 - spec/misc/pg_reserved.txt
 - spec/multibyte_spec.rb
+- spec/postgresql_spec.rb
 - spec/precision_spec.rb
 - spec/reserved_words_spec.rb
 - spec/sequel_spec.rb
@@ -326,6 +315,8 @@ files:
 - spec/threaded_spec.rb
 - spec/timezones_spec.rb
 - spec/type_safety_spec.rb
+- travis/install_postgres.sh
+- travis/tune_mysql.sh
 - upsert.gemspec
 homepage: https://github.com/seamusabshere/upsert
 licenses: []
@@ -346,7 +337,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: Make it easy to upsert on MySQL, PostgreSQL, and SQLite3. Transparently creates
@@ -358,11 +349,13 @@ test_files:
 - spec/database_functions_spec.rb
 - spec/database_spec.rb
 - spec/hstore_spec.rb
+- spec/jruby_spec.rb
 - spec/logger_spec.rb
 - spec/misc/get_postgres_reserved_words.rb
 - spec/misc/mysql_reserved.txt
 - spec/misc/pg_reserved.txt
 - spec/multibyte_spec.rb
+- spec/postgresql_spec.rb
 - spec/precision_spec.rb
 - spec/reserved_words_spec.rb
 - spec/sequel_spec.rb
@@ -371,4 +364,3 @@ test_files:
 - spec/threaded_spec.rb
 - spec/timezones_spec.rb
 - spec/type_safety_spec.rb
-has_rdoc: