upsert 2.1.0 → 2.9.10

data/lib/upsert/column_definition/sqlite3.rb

@@ -3,9 +3,9 @@ class Upsert
     # @private
     class Sqlite3 < ColumnDefinition
       class << self
-        def all(connection, table_name)
+        def all(connection, quoted_table_name)
           # activerecord-3.2.13/lib/active_record/connection_adapters/sqlite_adapter.rb
-          connection.execute("PRAGMA table_info(#{connection.quote_ident(table_name)})").map do |row|#, 'SCHEMA').to_hash
+          connection.execute("PRAGMA table_info(#{quoted_table_name})").map do |row|#, 'SCHEMA').to_hash
             if connection.metal.respond_to?(:results_as_hash) and not connection.metal.results_as_hash
               row = {'name' => row[1], 'type' => row[2], 'dflt_value' => row[4]}
             end
@@ -25,7 +25,7 @@ class Upsert
           end
         end
       end
-      
+
       def equality(left, right)
         "(#{left} IS #{right} OR (#{left} IS NULL AND #{right} IS NULL))"
       end

data/lib/upsert/connection/Java_ComMysqlJdbc_JDBC4Connection.rb

@@ -6,16 +6,22 @@ class Upsert
     class Java_ComMysqlJdbc_JDBC4Connection < Connection
       include Jdbc
 
-      # ? backtick?
       def quote_ident(k)
-        DOUBLE_QUOTE + k.to_s.gsub(DOUBLE_QUOTE, '""') + DOUBLE_QUOTE
+        if metal.useAnsiQuotedIdentifiers
+          DOUBLE_QUOTE + k.to_s.gsub(DOUBLE_QUOTE, '""') + DOUBLE_QUOTE
+        else
+          # Escape backticks by doubling them.  Ref http://dev.mysql.com/doc/refman/5.7/en/identifiers.html
+          BACKTICK + k.to_s.gsub(BACKTICK, BACKTICK + BACKTICK) + BACKTICK
+        end
       end
 
       def bind_value(v)
         case v
-        when Time, DateTime
-          # mysql doesn't like it when you send timezone to a datetime
-          Upsert.utc_iso8601 v, false
+        when DateTime, Time
+          date = v.utc
+          java.time.LocalDateTime.of(date.year, date.month, date.day, date.hour, date.min, date.sec, date.nsec)
+        when Date
+          java.time.LocalDate.of(v.year, v.month, v.day)
         else
           super
         end

data/lib/upsert/connection/Java_OrgPostgresqlJdbc_PgConnection.rb

@@ -0,0 +1,33 @@
+require_relative "jdbc"
+require_relative "postgresql"
+
+class Upsert
+  class Connection
+    # @private
+    class Java_OrgPostgresqlJdbc_PgConnection < Connection
+      include Jdbc
+      include Postgresql
+
+      def quote_ident(k)
+        DOUBLE_QUOTE + k.to_s.gsub(DOUBLE_QUOTE, '""') + DOUBLE_QUOTE
+      end
+
+      def in_transaction?
+        # https://github.com/kares/activerecord-jdbc-adapter/commit/4d6e0e0c52d12b0166810dffc9f898141a23bee6
+        ![0, 4].include?(metal.get_transaction_state)
+      end
+
+      def bind_value(v)
+        case v
+        when DateTime, Time
+          date = v.utc
+          java.time.LocalDateTime.of(date.year, date.month, date.day, date.hour, date.min, date.sec, date.nsec)
+        when Date
+          java.time.LocalDate.of(v.year, v.month, v.day)
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/upsert/connection/PG_Connection.rb

@@ -1,12 +1,17 @@
+require_relative "postgresql"
+
 class Upsert
   class Connection
     # @private
     class PG_Connection < Connection
       include Postgresql
-      
+
       def execute(sql, params = nil)
         if params
           # Upsert.logger.debug { %{[upsert] #{sql} with #{params.inspect}} }
+          # The following will blow up if you pass a value that cannot be automatically type-casted,
+          #   such as passing a string to an integer field.  You'll get an error something along the
+          #   lines of: "invalid input syntax for <type>: <value>"
           metal.exec sql, convert_binary(params)
         else
           Upsert.logger.debug { %{[upsert] #{sql}} }
@@ -21,6 +26,10 @@ class Upsert
       def binary(v)
         { :value => v.value, :format => 1 }
       end
+
+      def in_transaction?
+        ![PG::PQTRANS_IDLE, PG::PQTRANS_UNKNOWN].include?(metal.transaction_status)
+      end
     end
   end
 end

data/lib/upsert/connection/jdbc.rb

@@ -4,11 +4,15 @@ class Upsert
     module Jdbc
       # /Users/seamusabshere/.rvm/gems/jruby-head/gems/activerecord-jdbc-adapter-1.2.2.1/src/java/arjdbc/jdbc/RubyJdbcConnection.java
       GETTER = {
+        java.sql.Types::CHAR        => 'getString',
         java.sql.Types::VARCHAR     => 'getString',
         java.sql.Types::OTHER       => 'getString', # ?! i guess unicode text?
         java.sql.Types::BINARY      => 'getBlob',
         java.sql.Types::LONGVARCHAR => 'getString',
+        java.sql.Types::BIGINT      => 'getLong',
         java.sql.Types::INTEGER     => 'getInt',
+        java.sql.Types::REAL        => "getLong",
+        java.sql.Types::ARRAY       => ->(r, i){ r.getArray(i).array.to_ary }
       }
       java.sql.Types.constants.each do |type_name|
         i = java.sql.Types.const_get type_name
@@ -22,6 +26,7 @@ class Upsert
         'TrueClass'  => 'setBoolean',
         'FalseClass' => 'setBoolean',
         'Fixnum'     => 'setInt',
+        'Integer'    => 'setInt'
       )
 
       def binary(v)
@@ -34,16 +39,24 @@ class Upsert
           setters = self.class.const_get(:SETTER)
           statement = metal.prepareStatement sql
           params.each_with_index do |v, i|
+            if v.is_a?(Fixnum) && v > 2_147_483_647
+              statement.setLong i+1, v
+              next
+            end
+
             case v
             when Upsert::Binary
               statement.setBytes i+1, binary(v)
-            when BigDecimal
+            when Float, BigDecimal
               statement.setBigDecimal i+1, java.math.BigDecimal.new(v.to_s)
             when NilClass
               # http://stackoverflow.com/questions/4243513/why-does-preparedstatement-setnull-requires-sqltype
               statement.setObject i+1, nil
+            when java.time.LocalDateTime, java.time.Instant, java.time.LocalDate
+              statement.setObject i+1, v
             else
               setter = setters[v.class.name]
+              Upsert.logger.debug { "Setting [#{v.class}, #{v}] via #{setter}" }
               statement.send setter, i+1, v
             end
           end
@@ -72,6 +85,8 @@ class Upsert
             column_name, getter = cg
             if getter == 'getNull'
               row[column_name] = nil
+            elsif getter.respond_to?(:call)
+              row[column_name] = getter.call(raw_result, i)
             else
               row[column_name] = raw_result.send(getter, i)
             end
@@ -81,6 +96,10 @@ class Upsert
         statement.close
         result
       end
+
+      def in_transaction?
+        raise "Not implemented"
+      end
     end
   end
 end

data/lib/upsert/connection/postgresql.rb

@@ -8,9 +8,8 @@ class Upsert
           # pg array escaping lifted from https://github.com/tlconnor/activerecord-postgres-array/blob/master/lib/activerecord-postgres-array/array.rb
           '{' + v.map do |vv|
             vv = vv.to_s.dup
-            vv.gsub! /\\/, '\&\&'
-            vv.gsub! /'/, "''"
-            vv.gsub! /"/, '\"'
+            vv.gsub!(/\\/, '\&\&')
+            vv.gsub!(/"/, '\"')
             %{"#{vv}"}
           end.join(',') + '}'
         when Hash

data/lib/upsert/merge_function.rb

@@ -11,11 +11,11 @@ class Upsert
       def unique_name(table_name, selector_keys, setter_keys)
         parts = [
           NAME_PREFIX,
-          table_name,
+          [*table_name].join("_").gsub(/[^\w_]+/, "_"),
           'SEL',
-          selector_keys.join('_A_'),
+          selector_keys.join('_A_').gsub(" ","_"),
           'SET',
-          setter_keys.join('_A_')
+          setter_keys.join('_A_').gsub(" ","_")
         ].join('_')
         if parts.length > MAX_NAME_LENGTH
           # maybe i should md5 instead
@@ -35,8 +35,9 @@ class Upsert
       @controller = controller
       @selector_keys = selector_keys
       @setter_keys = setter_keys
+      @assume_function_exists = assume_function_exists
       validate!
-      create! unless assume_function_exists
+      create! unless @assume_function_exists
     end
 
     def name

data/lib/upsert/merge_function/Java_OrgPostgresqlJdbc_PgConnection.rb

@@ -0,0 +1,27 @@
+require 'upsert/merge_function/postgresql'
+
+class Upsert
+  class MergeFunction
+    # @private
+    class Java_OrgPostgresqlJdbc_PgConnection < MergeFunction
+      ERROR_CLASS = org.postgresql.util.PSQLException
+      include Postgresql
+
+      def execute_parameterized(query, args = [])
+        query_args = []
+        query = query.gsub(/\$(\d+)/) do |str|
+          query_args << args[Regexp.last_match[1].to_i - 1]
+          "?"
+        end
+        controller.connection.execute(query, query_args)
+      end
+
+      def unique_index_on_selector?
+        return @unique_index_on_selector if defined?(@unique_index_on_selector)
+        @unique_index_on_selector = unique_index_columns.any? do |row|
+          row["index_columns"].sort == selector_keys.sort
+        end
+      end
+    end
+  end
+end

data/lib/upsert/merge_function/PG_Connection.rb

@@ -4,52 +4,21 @@ class Upsert
   class MergeFunction
     # @private
     class PG_Connection < MergeFunction
+      ERROR_CLASS = PG::Error
       include Postgresql
 
-      def execute(row)
-        first_try = true
-        values = []
-        values += row.selector.values
-        values += row.setter.values
-        hstore_delete_handlers.each do |hstore_delete_handler|
-          values << row.hstore_delete_keys.fetch(hstore_delete_handler.name, [])
-        end
-        Upsert.logger.debug do
-          %{[upsert]\n\tSelector: #{row.selector.inspect}\n\tSetter: #{row.setter.inspect}}
-        end
-        begin
-          connection.execute sql, values.map { |v| connection.bind_value v }
-        rescue PG::Error => pg_error
-          if pg_error.message =~ /function #{name}.* does not exist/i
-            if first_try
-              Upsert.logger.info %{[upsert] Function #{name.inspect} went missing, trying to recreate}
-              first_try = false
-              create!
-              retry
-            else
-              Upsert.logger.info %{[upsert] Failed to create function #{name.inspect} for some reason}
-              raise pg_error
-            end
-          else
-            raise pg_error
-          end
-        end
+      def execute_parameterized(query, args = [])
+        controller.connection.execute(query, args)
       end
 
-      # strangely ? can't be used as a placeholder
-      def sql
-        @sql ||= begin
-          bind_params = []
-          i = 1
-          (selector_keys.length + setter_keys.length).times do
-            bind_params << "$#{i}"
-            i += 1
-          end
-          hstore_delete_handlers.length.times do
-            bind_params << "$#{i}::text[]"
-            i += 1
-          end
-          %{SELECT #{name}(#{bind_params.join(', ')})}
+      def unique_index_on_selector?
+        return @unique_index_on_selector if defined?(@unique_index_on_selector)
+
+        type_map = PG::TypeMapByColumn.new([PG::TextDecoder::Array.new])
+        res = unique_index_columns.tap { |r| r.type_map = type_map }
+
+        @unique_index_on_selector = res.values.any? do |row|
+          row.first.sort == selector_keys.sort
         end
       end
     end

data/lib/upsert/merge_function/postgresql.rb

@@ -40,13 +40,227 @@ class Upsert
         end
       end
 
+      attr_reader :quoted_setter_names
+      attr_reader :quoted_selector_names
+
+      def initialize(controller, *args)
+        super
+        @quoted_setter_names = setter_keys.map { |k| connection.quote_ident k }
+        @quoted_selector_names = selector_keys.map { |k| connection.quote_ident k }
+      end
+
+      def execute(row)
+        use_pg_native? ? pg_native(row) : pg_function(row)
+      end
+
+      def pg_function(row)
+        values = []
+        values += row.selector.values
+        values += row.setter.values
+        hstore_delete_handlers.each do |hstore_delete_handler|
+          values << row.hstore_delete_keys.fetch(hstore_delete_handler.name, [])
+        end
+        Upsert.logger.debug do
+          %{[upsert]\n\tSelector: #{row.selector.inspect}\n\tSetter: #{row.setter.inspect}}
+        end
+
+        first_try = true
+        begin
+          create! if !@assume_function_exists && (connection.in_transaction? && !function_exists?)
+          execute_parameterized(sql, values.map { |v| connection.bind_value v })
+        rescue self.class::ERROR_CLASS => pg_error
+          if pg_error.message =~ /function #{name}.* does not exist/i
+            if first_try
+              Upsert.logger.info %{[upsert] Function #{name.inspect} went missing, trying to recreate}
+              first_try = false
+              create!
+              retry
+            end
+            Upsert.logger.info %{[upsert] Failed to create function #{name.inspect} for some reason}
+            raise pg_error
+          else
+            raise pg_error
+          end
+        end
+      end
+
+      def function_exists?
+        @function_exists ||= controller.connection.execute("SELECT count(*) AS cnt FROM pg_proc WHERE lower(proname) = lower('#{name}')").first["cnt"].to_i > 0
+      end
+
+      # strangely ? can't be used as a placeholder
       def sql
         @sql ||= begin
-          bind_params = Array.new(selector_keys.length + setter_keys.length, '?') + Array.new(hstore_delete_handlers.length, '?::text[]')
+          bind_params = []
+          i = 1
+          (selector_keys.length + setter_keys.length).times do
+            bind_params << "$#{i}"
+            i += 1
+          end
+          hstore_delete_handlers.length.times do
+            bind_params << "$#{i}::text[]"
+            i += 1
+          end
           %{SELECT #{name}(#{bind_params.join(', ')})}
         end
       end
 
+      def use_pg_native?
+        return @use_pg_native if defined?(@use_pg_native)
+
+        @use_pg_native = server_version >= 90500 && unique_index_on_selector?
+        Upsert.logger.warn "[upsert] WARNING: Not using native PG CONFLICT / UPDATE" unless @use_pg_native
+        @use_pg_native
+      end
+
+      def server_version
+        @server_version ||= Upsert::MergeFunction::Postgresql.extract_version(
+          controller.connection.execute("SHOW server_version").first["server_version"]
+        )
+      end
+
+      # Extracted from https://github.com/dr-itz/activerecord-jdbc-adapter/blob/master/lib/arjdbc/postgresql/adapter.rb
+      def self.extract_version(version_string)
+        # Use the same versioning format as jdbc-postgresql and libpq
+        # https://github.com/dr-itz/activerecord-jdbc-adapter/commit/fd79756374c62fa9d009995dd1914d780e6a3dbf
+        # https://github.com/postgres/postgres/blob/master/src/interfaces/libpq/fe-exec.c
+        if (match = version_string.match(/([\d\.]*\d).*?/))
+          version = match[1].split('.').map(&:to_i)
+          # PostgreSQL version representation does not have more than 4 digits
+          # From version 10 onwards, PG has changed its versioning policy to
+          # limit it to only 2 digits. i.e. in 10.x, 10 being the major
+          # version and x representing the patch release
+          # Refer to:
+          #   https://www.postgresql.org/support/versioning/
+          #   https://www.postgresql.org/docs/10/static/libpq-status.html -> PQserverVersion()
+          # for more info
+
+          if version.size >= 3
+            (version[0] * 100 + version[1]) * 100 + version[2]
+          elsif version.size == 2
+            if version[0] >= 10
+              version[0] * 100 * 100 + version[1]
+            else
+              (version[0] * 100 + version[1]) * 100
+            end
+          elsif version.size == 1
+            version[0] * 100 * 100
+          else
+            0
+          end
+        else
+          0
+        end
+      end
+
+      def unique_index_columns
+        if table_name.is_a?(Array) && table_name.length > 1
+          schema_argument = '$2'
+          table_name_arguments = table_name
+        else
+          schema_argument = 'ANY(current_schemas(true)::text[])'
+          table_name_arguments = [*table_name]
+        end
+
+        table_name_arguments.reverse!
+
+        execute_parameterized(
+          %{
+            SELECT
+            ARRAY(
+              SELECT pg_get_indexdef(pg_index.indexrelid, k + 1, TRUE)
+              FROM
+                generate_subscripts(pg_index.indkey, 1) AS k
+              ORDER BY k
+            ) AS index_columns
+              FROM pg_index
+              JOIN pg_class AS idx ON idx.oid = pg_index.indexrelid
+              JOIN pg_class AS tbl ON tbl.oid = pg_index.indrelid
+              JOIN pg_namespace ON pg_namespace.oid = idx.relnamespace
+            WHERE pg_index.indisunique IS TRUE AND pg_namespace.nspname = #{schema_argument} AND tbl.relname = $1
+          },
+          table_name_arguments
+        )
+      end
+
+      def pg_native(row)
+        bind_setter_values = row.setter.values.map { |v| connection.bind_value v }
+        # TODO: Is this needed?
+        row_syntax = server_version >= 100 ? "ROW" : ""
+
+        upsert_sql = %{
+          INSERT INTO #{quoted_table_name} (#{quoted_setter_names.join(',')})
+          VALUES (#{insert_bind_placeholders(row).join(', ')})
+          ON CONFLICT(#{quoted_selector_names.join(', ')})
+          DO UPDATE SET #{quoted_setter_names.zip(conflict_bind_placeholders(row)).map { |n, v| "#{n} = #{v}" }.join(', ')}
+        }
+
+        execute_parameterized(upsert_sql, bind_setter_values)
+      end
+
+      def hstore_delete_function(sql, row, column_definition)
+        parts = []
+        if row.hstore_delete_keys.key?(column_definition.name)
+          parts << "DELETE("
+        end
+        parts << sql
+        if row.hstore_delete_keys.key?(column_definition.name)
+          keys = row.hstore_delete_keys[column_definition.name].map { |k| "'#{k.to_s.gsub("'", "\\'")}'" }
+          parts << ", ARRAY[#{keys.join(', ')}])"
+        end
+
+        parts.join(" ")
+      end
+
+      def insert_bind_placeholders(row)
+        if row.hstore_delete_keys.empty?
+          @insert_bind_placeholders ||= setter_column_definitions.each_with_index.map do |column_definition, i|
+            if column_definition.hstore?
+              "CAST($#{i + 1} AS hstore)"
+            else
+              "$#{i + 1}"
+            end
+          end
+        else
+          setter_column_definitions.each_with_index.map do |column_definition, i|
+            idx = i + 1
+            if column_definition.hstore?
+              hstore_delete_function("CAST($#{idx} AS hstore)", row, column_definition)
+            else
+              "$#{idx}"
+            end
+          end
+        end
+      end
+
+      def conflict_bind_placeholders(row)
+        if row.hstore_delete_keys.empty?
+          @conflict_bind_placeholders ||= setter_column_definitions.each_with_index.map do |column_definition, i|
+            idx = i + 1
+            if column_definition.hstore?
+              "CASE WHEN #{quoted_table_name}.#{column_definition.quoted_name} IS NULL THEN CAST($#{idx} AS hstore) ELSE" \
+                + " (#{quoted_table_name}.#{column_definition.quoted_name} || CAST($#{idx} AS hstore))" \
+                + " END"
+            else
+              "$#{idx}"
+            end
+          end
+        else
+          setter_column_definitions.each_with_index.map do |column_definition, i|
+            idx = i + 1
+            if column_definition.hstore?
+              "CASE WHEN #{quoted_table_name}.#{column_definition.quoted_name} IS NULL THEN " \
+                + hstore_delete_function("CAST($#{idx} AS hstore)", row, column_definition) \
+                + " ELSE " \
+                + hstore_delete_function("(#{quoted_table_name}.#{column_definition.quoted_name} || CAST($#{idx} AS hstore))", row, column_definition) \
+                + " END"
+            else
+              "$#{idx}"
+            end
+          end
+        end
+      end
+
       class HstoreDeleteHandler
         attr_reader :merge_function
         attr_reader :column_definition