upsert 0.5.0 → 1.0.2

data/lib/upsert/cell.rb

@@ -1,7 +1,3 @@
-require 'upsert/cell/mysql2_client'
-require 'upsert/cell/pg_connection'
-require 'upsert/cell/sqlite3_database'
-
 class Upsert
   # @private
   class Cell

data/lib/upsert/column_definition.rb

@@ -0,0 +1,43 @@
+class Upsert
+  # @private
+  class ColumnDefinition
+    class << self
+      # activerecord-3.2.X/lib/active_record/connection_adapters/XXXXXXXXX_adapter.rb#column_definitions
+      def all(connection, table_name)
+        raise "not impl"
+      end
+    end
+
+    attr_reader :name
+    attr_reader :sql_type
+    attr_reader :default
+    attr_reader :quoted_name
+    attr_reader :quoted_selector_name
+    attr_reader :quoted_setter_name
+
+    def initialize(connection, name, sql_type, default)
+      @name = name
+      @sql_type = sql_type
+      @default = default
+      @quoted_name = connection.quote_ident name
+      @quoted_selector_name = connection.quote_ident "#{name}_sel"
+      @quoted_setter_name = connection.quote_ident "#{name}_set"
+    end
+
+    def to_selector_arg
+      "#{quoted_selector_name} #{sql_type}"
+    end
+
+    def to_setter_arg
+      "#{quoted_setter_name} #{sql_type}"
+    end
+
+    def to_setter
+      "#{quoted_name} = #{quoted_setter_name}"
+    end
+
+    def to_selector
+      "#{quoted_name} = #{quoted_selector_name}"
+    end
+  end
+end

data/lib/upsert/column_definition/Mysql2_Client.rb

@@ -0,0 +1,24 @@
+class Upsert
+  class ColumnDefinition
+    # @private
+    class Mysql2_Client < ColumnDefinition
+      class << self
+        def all(connection, table_name)
+          connection.execute("SHOW COLUMNS FROM #{connection.quote_ident(table_name)}").map do |row|
+            name, type, default = if row.is_a?(Array)
+              # you don't know if mysql2 is going to give you an array or a hash... and you shouldn't specify, because it's sticky
+              # ["name", "varchar(255)", "YES", "UNI", nil, ""]
+              row.values_at(0,1,4)
+            else
+              # {"Field"=>"name", "Type"=>"varchar(255)", "Null"=>"NO", "Key"=>"PRI", "Default"=>nil, "Extra"=>""}
+              [row['Field'], row['Type'], row['Default']]
+            end
+            new connection, name, type, default
+          end.sort_by do |cd|
+            cd.name
+          end
+        end
+      end
+    end
+  end
+end

data/lib/upsert/column_definition/PG_Connection.rb

@@ -0,0 +1,24 @@
+class Upsert
+  class ColumnDefinition
+    # @private
+    class PG_Connection < ColumnDefinition
+      class << self
+        # activerecord-3.2.5/lib/active_record/connection_adapters/postgresql_adapter.rb#column_definitions
+        def all(connection, table_name)
+          res = connection.execute <<-EOS
+SELECT a.attname AS name, format_type(a.atttypid, a.atttypmod) AS sql_type, d.adsrc AS default
+FROM pg_attribute a LEFT JOIN pg_attrdef d
+  ON a.attrelid = d.adrelid AND a.attnum = d.adnum
+WHERE a.attrelid = '#{connection.quote_ident(table_name)}'::regclass
+AND a.attnum > 0 AND NOT a.attisdropped
+EOS
+          res.map do |row|
+            new connection, row['name'], row['sql_type'], row['default']
+          end.sort_by do |cd|
+            cd.name
+          end
+        end
+      end
+    end
+  end
+end

data/lib/upsert/column_definition/SQLite3_Database.rb

@@ -0,0 +1,7 @@
+class Upsert
+  class ColumnDefinition
+    # @private
+    class SQLite3_Database < ColumnDefinition
+    end
+  end
+end

data/lib/upsert/connection.rb

@@ -1,15 +1,11 @@
-require 'upsert/connection/mysql2_client'
-require 'upsert/connection/pg_connection'
-require 'upsert/connection/sqlite3_database'
-
 class Upsert
   # @private
   class Connection
-    attr_reader :parent
+    attr_reader :controller
     attr_reader :raw_connection
 
-    def initialize(parent, raw_connection)
-      @parent = parent
+    def initialize(controller, raw_connection)
+      @controller = controller
       @raw_connection = raw_connection
     end
     

data/lib/upsert/merge_function.rb

@@ -0,0 +1,72 @@
+require 'zlib'
+
+class Upsert
+  # @private
+  class MergeFunction
+    MAX_NAME_LENGTH = 63
+
+    class << self
+      def execute(controller, row)
+        merge_function = lookup controller, row
+        merge_function.execute row
+      end
+
+      def unique_name(table_name, selector_keys, setter_keys)
+        parts = [
+          'upsert',
+          table_name,
+          'SEL',
+          selector_keys.join('_A_'),
+          'SET',
+          setter_keys.join('_A_')
+        ].join('_')
+        if parts.length > MAX_NAME_LENGTH
+          # maybe i should md5 instead
+          crc32 = Zlib.crc32(parts).to_s
+          [ parts.first(MAX_NAME_LENGTH-11), crc32 ].join
+        else
+          parts
+        end
+      end
+
+      def lookup(controller, row)
+        @lookup ||= {}
+        selector_keys = row.selector.keys
+        setter_keys = row.setter.keys
+        key = [controller.table_name, selector_keys, setter_keys]
+        @lookup[key] ||= new(controller, selector_keys, setter_keys)
+      end
+    end
+
+    attr_reader :controller
+    attr_reader :selector_keys
+    attr_reader :setter_keys
+
+    def initialize(controller, selector_keys, setter_keys)
+      @controller = controller
+      @selector_keys = selector_keys
+      @setter_keys = setter_keys
+      create!
+    end
+
+    def name
+      @name ||= MergeFunction.unique_name table_name, selector_keys, setter_keys
+    end
+
+    def connection
+      controller.connection
+    end
+
+    def table_name
+      controller.table_name
+    end
+
+    def quoted_table_name
+      controller.quoted_table_name
+    end
+
+    def column_definitions
+      controller.column_definitions
+    end
+  end
+end

data/lib/upsert/merge_function/Mysql2_Client.rb

@@ -0,0 +1,89 @@
+require 'digest/md5'
+
+class Upsert
+  class MergeFunction
+    # @private
+    class Mysql2_Client < MergeFunction
+      MAX_NAME_LENGTH = 63
+
+      class << self
+        # http://stackoverflow.com/questions/733349/list-of-stored-procedures-functions-mysql-command-line
+        def clear(connection)
+          connection.execute("SHOW PROCEDURE STATUS WHERE Db = DATABASE() AND Name LIKE 'upsert_%'").map { |row| row['Name'] }.each do |name|
+            connection.execute "DROP PROCEDURE IF EXISTS #{connection.quote_ident(name)}"
+          end
+        end
+      end
+
+      def execute(row)
+        first_try = true
+        begin
+          connection.execute sql(row)
+        rescue Mysql2::Error => e
+          if e.message =~ /PROCEDURE.*does not exist/i
+            if first_try
+              Upsert.logger.info %{[upsert] Function #{name.inspect} went missing, trying to recreate}
+              first_try = false
+              create!
+              retry
+            else
+              Upsert.logger.info %{[upsert] Failed to create function #{name.inspect} for some reason}
+              raise e
+            end
+          else
+            raise e
+          end
+        end
+      end
+
+      def sql(row)
+        quoted_params = (row.selector.values + row.setter.values).map(&:quoted_value)
+        %{CALL #{name}(#{quoted_params.join(', ')})}
+      end
+
+      # http://stackoverflow.com/questions/11371479/how-to-translate-postgresql-merge-db-aka-upsert-function-into-mysql/
+      def create!
+        Upsert.logger.info "[upsert] Creating or replacing database function #{name.inspect} on table #{table_name.inspect} for selector #{selector_keys.map(&:inspect).join(', ')} and setter #{setter_keys.map(&:inspect).join(', ')}"
+        selector_column_definitions = column_definitions.select { |cd| selector_keys.include?(cd.name) }
+        setter_column_definitions = column_definitions.select { |cd| setter_keys.include?(cd.name) }
+        quoted_name = connection.quote_ident name
+        connection.execute "DROP PROCEDURE IF EXISTS #{quoted_name}"
+        connection.execute(%{
+          CREATE PROCEDURE #{quoted_name}(#{(selector_column_definitions.map(&:to_selector_arg) + setter_column_definitions.map(&:to_setter_arg)).join(', ')})
+          BEGIN
+            DECLARE done BOOLEAN;
+            REPEAT
+              BEGIN
+                -- If there is a unique key constraint error then 
+                -- someone made a concurrent insert. Reset the sentinel
+                -- and try again.
+                DECLARE ER_DUP_UNIQUE CONDITION FOR 23000;
+                DECLARE ER_INTEG CONDITION FOR 1062;
+                DECLARE CONTINUE HANDLER FOR ER_DUP_UNIQUE BEGIN
+                  SET done = FALSE;
+                END;
+                
+                DECLARE CONTINUE HANDLER FOR ER_INTEG BEGIN
+                  SET done = TRUE;
+                END;
+
+                SET done = TRUE;
+                SELECT COUNT(*) INTO @count FROM #{quoted_table_name} WHERE #{selector_column_definitions.map(&:to_selector).join(' AND ')};
+                -- Race condition here. If a concurrent INSERT is made after
+                -- the SELECT but before the INSERT below we'll get a duplicate
+                -- key error. But the handler above will take care of that.
+                IF @count > 0 THEN 
+                  -- UPDATE table_name SET b = b_SET WHERE a = a_SEL;
+                  UPDATE #{quoted_table_name} SET #{setter_column_definitions.map(&:to_setter).join(', ')} WHERE #{selector_column_definitions.map(&:to_selector).join(' AND ')};
+                ELSE
+                  -- INSERT INTO table_name (a, b) VALUES (k, data);
+                  INSERT INTO #{quoted_table_name} (#{setter_column_definitions.map(&:quoted_name).join(', ')}) VALUES (#{setter_column_definitions.map(&:quoted_setter_name).join(', ')});
+                END IF;
+              END;
+            UNTIL done END REPEAT;
+          END
+        })
+      end
+    end
+  end
+end

data/lib/upsert/merge_function/PG_Connection.rb

@@ -0,0 +1,114 @@
+class Upsert
+  class MergeFunction
+    # @private
+    class PG_Connection < MergeFunction
+      MAX_NAME_LENGTH = 63
+
+      class << self
+        def clear(connection)
+          # http://stackoverflow.com/questions/7622908/postgresql-drop-function-without-knowing-the-number-type-of-parameters
+          connection.execute(%{
+            CREATE OR REPLACE FUNCTION pg_temp.upsert_delfunc(text)
+              RETURNS void AS
+            $BODY$
+            DECLARE
+              _sql text;
+            BEGIN
+            FOR _sql IN
+              SELECT 'DROP FUNCTION ' || quote_ident(n.nspname)
+                               || '.' || quote_ident(p.proname)
+                               || '(' || pg_catalog.pg_get_function_identity_arguments(p.oid) || ');'
+              FROM   pg_catalog.pg_proc p
+              LEFT   JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
+              WHERE  p.proname = $1
+              AND    pg_catalog.pg_function_is_visible(p.oid) -- you may or may not want this
+            LOOP
+              EXECUTE _sql;
+            END LOOP;
+            END;
+            $BODY$
+              LANGUAGE plpgsql;
+          })
+          connection.execute(%{SELECT proname FROM pg_proc WHERE proname LIKE 'upsert_%'}).each do |row|
+            k = row['proname']
+            next if k == 'upsert_delfunc'
+            Upsert.logger.info %{[upsert] Dropping function #{k.inspect}}
+            connection.execute %{SELECT pg_temp.upsert_delfunc('#{k}')}
+          end
+        end
+      end
+
+      def execute(row)
+        first_try = true
+        bind_selector_values = row.selector.values.map(&:bind_value)
+        bind_setter_values = row.setter.values.map(&:bind_value)
+        begin
+          connection.execute sql, (bind_selector_values + bind_setter_values)
+        rescue PG::Error => pg_error
+          if pg_error.message =~ /function #{name}.* does not exist/i
+            if first_try
+              Upsert.logger.info %{[upsert] Function #{name.inspect} went missing, trying to recreate}
+              first_try = false
+              create!
+              retry
+            else
+              Upsert.logger.info %{[upsert] Failed to create function #{name.inspect} for some reason}
+              raise pg_error
+            end
+          else
+            raise pg_error
+          end
+        end
+      end
+
+      def sql
+        @sql ||= begin
+          bind_params = []
+          1.upto(selector_keys.length + setter_keys.length) { |i| bind_params << "$#{i}" }
+          %{SELECT #{name}(#{bind_params.join(', ')})}
+        end
+      end
+
+      # the "canonical example" from http://www.postgresql.org/docs/9.1/static/plpgsql-control-structures.html#PLPGSQL-UPSERT-EXAMPLE
+      # differentiate between selector and setter
+      def create!
+        Upsert.logger.info "[upsert] Creating or replacing database function #{name.inspect} on table #{table_name.inspect} for selector #{selector_keys.map(&:inspect).join(', ')} and setter #{setter_keys.map(&:inspect).join(', ')}"
+        selector_column_definitions = column_definitions.select { |cd| selector_keys.include?(cd.name) }
+        setter_column_definitions = column_definitions.select { |cd| setter_keys.include?(cd.name) }
+        connection.execute(%{
+          CREATE OR REPLACE FUNCTION #{name}(#{(selector_column_definitions.map(&:to_selector_arg) + setter_column_definitions.map(&:to_setter_arg)).join(', ')}) RETURNS VOID AS
+          $$
+          DECLARE
+            first_try INTEGER := 1;
+          BEGIN
+            LOOP
+              -- first try to update the key
+              UPDATE #{quoted_table_name} SET #{setter_column_definitions.map(&:to_setter).join(', ')}
+                WHERE #{selector_column_definitions.map(&:to_selector).join(' AND ') };
+              IF found THEN
+                RETURN;
+              END IF;
+              -- not there, so try to insert the key
+              -- if someone else inserts the same key concurrently,
+              -- we could get a unique-key failure
+              BEGIN
+                INSERT INTO #{quoted_table_name}(#{setter_column_definitions.map(&:quoted_name).join(', ')}) VALUES (#{setter_column_definitions.map(&:quoted_setter_name).join(', ')});
+                RETURN;
+              EXCEPTION WHEN unique_violation THEN
+                -- seamusabshere 9/20/12 only retry once
+                IF (first_try = 1) THEN
+                  first_try := 0;
+                ELSE
+                  RETURN;
+                END IF;
+                -- Do nothing, and loop to try the UPDATE again.
+              END;
+            END LOOP;
+          END;
+          $$
+          LANGUAGE plpgsql;
+        })
+      end
+    end
+  end
+end

data/lib/upsert/merge_function/SQLite3_Database.rb

@@ -0,0 +1,29 @@
+class Upsert
+  class MergeFunction
+    # @private
+    class SQLite3_Database < MergeFunction
+      attr_reader :quoted_setter_names
+      attr_reader :quoted_selector_names
+
+      def initialize(*)
+        super
+        @quoted_setter_names = setter_keys.map { |k| connection.quote_ident k }
+        @quoted_selector_names = selector_keys.map { |k| connection.quote_ident k }
+      end
+
+      def create!
+        # not necessary
+      end
+
+      def execute(row)
+        bind_setter_values = row.setter.values.map(&:bind_value)
+        
+        insert_or_ignore_sql = %{INSERT OR IGNORE INTO #{quoted_table_name} (#{quoted_setter_names.join(',')}) VALUES (#{Array.new(bind_setter_values.length, '?').join(',')})}
+        connection.execute insert_or_ignore_sql, bind_setter_values
+
+        update_sql = %{UPDATE #{quoted_table_name} SET #{quoted_setter_names.map { |qk| "#{qk}=?" }.join(',')} WHERE #{quoted_selector_names.map { |qk| "#{qk}=?" }.join(' AND ')}}
+        connection.execute update_sql, (bind_setter_values + row.selector.values.map(&:bind_value))
+      end
+    end
+  end
+end