rubyrep 1.0.0

data/lib/rubyrep/replication_extenders/mysql_replication.rb

@@ -0,0 +1,271 @@
+module RR
+  module ReplicationExtenders
+
+    # Provides Mysql specific functionality for database replication
+    module MysqlReplication
+      RR::ReplicationExtenders.register :mysql => self
+
+      # Creates or replaces the replication trigger function.
+      # See #create_replication_trigger for a descriptions of the +params+ hash.
+      def create_or_replace_replication_trigger_function(params)
+        execute(<<-end_sql)
+          DROP PROCEDURE IF EXISTS #{params[:trigger_name]};
+        end_sql
+        
+        activity_check = ""
+        if params[:exclude_rr_activity] then
+          activity_check = <<-end_sql
+            DECLARE active INT;
+            SELECT count(*) INTO active FROM #{params[:activity_table]};
+            IF active <> 0 THEN
+              LEAVE p;
+            END IF;
+          end_sql
+        end
+
+        execute(<<-end_sql)
+          CREATE PROCEDURE #{params[:trigger_name]}(change_key varchar(2000), change_new_key varchar(2000), change_type varchar(1))
+          p: BEGIN
+            #{activity_check}
+            INSERT INTO #{params[:log_table]}(change_table, change_key, change_new_key, change_type, change_time)
+              VALUES('#{params[:table]}', change_key, change_new_key, change_type, now());
+          END;
+        end_sql
+        
+      end
+
+      # Returns the key clause that is used in the trigger function.
+      # * +trigger_var+: should be either 'NEW' or 'OLD'
+      # * +params+: the parameter hash as described in #create_rep_trigger
+      def key_clause(trigger_var, params)
+        "concat_ws('#{params[:key_sep]}', " +
+          params[:keys].map { |key| "'#{key}', #{trigger_var}.#{key}"}.join(", ") +
+          ")"
+      end
+      private :key_clause
+
+      # Creates a trigger to log all changes for the given table.
+      # +params+ is a hash with all necessary information:
+      # * :+trigger_name+: name of the trigger
+      # * :+table+: name of the table that should be monitored
+      # * :+keys+: array of names of the key columns of the monitored table
+      # * :+log_table+: name of the table receiving all change notifications
+      # * :+activity_table+: name of the table receiving the rubyrep activity information
+      # * :+key_sep+: column seperator to be used in the key column of the log table
+      # * :+exclude_rr_activity+:
+      #   if true, the trigger will check and filter out changes initiated by RubyRep
+      def create_replication_trigger(params)
+        create_or_replace_replication_trigger_function params
+
+        %w(insert update delete).each do |action|
+          execute(<<-end_sql)
+            DROP TRIGGER IF EXISTS #{params[:trigger_name]}_#{action};
+          end_sql
+
+          # The created triggers can handle the case where the trigger procedure
+          # is updated (that is: temporarily deleted and recreated) while the
+          # trigger is running.
+          # For that an MySQL internal exception is raised if the trigger
+          # procedure cannot be found. The exception is caught by an trigger
+          # internal handler. 
+          # The handler causes the trigger to retry calling the
+          # trigger procedure several times with short breaks in between.
+
+          trigger_var = action == 'delete' ? 'OLD' : 'NEW'
+          if action == 'update'
+            call_statement = "CALL #{params[:trigger_name]}(#{key_clause('OLD', params)}, #{key_clause('NEW', params)}, '#{action[0,1].upcase}');"
+          else
+            call_statement = "CALL #{params[:trigger_name]}(#{key_clause(trigger_var, params)}, null, '#{action[0,1].upcase}');"
+          end
+          execute(<<-end_sql)
+            CREATE TRIGGER #{params[:trigger_name]}_#{action}
+              AFTER #{action} ON #{params[:table]} FOR EACH ROW BEGIN
+                DECLARE number_attempts INT DEFAULT 0;
+                DECLARE failed INT;
+                DECLARE CONTINUE HANDLER FOR 1305 BEGIN
+                  DO SLEEP(0.05);
+                  SET failed = 1;
+                  SET number_attempts = number_attempts + 1;
+                END;
+                REPEAT
+                  SET failed = 0;
+                  #{call_statement}
+                UNTIL failed = 0 OR number_attempts >= 40 END REPEAT;
+              END;
+          end_sql
+        end
+
+      end
+
+      # Removes a trigger and related trigger procedure.
+      # * +trigger_name+: name of the trigger
+      # * +table_name+: name of the table for which the trigger exists
+      def drop_replication_trigger(trigger_name, table_name)
+        %w(insert update delete).each do |action|
+          execute "DROP TRIGGER #{trigger_name}_#{action};"
+        end
+        execute "DROP PROCEDURE #{trigger_name};"
+      end
+
+      # Returns +true+ if the named trigger exists for the named table.
+      # * +trigger_name+: name of the trigger
+      # * +table_name+: name of the table
+      def replication_trigger_exists?(trigger_name, table_name)
+        !select_all("select 1 from information_schema.triggers where trigger_schema = database() and trigger_name = '#{trigger_name}_insert' and event_object_table = '#{table_name}'").empty?
+      end
+
+      # Returns all unadjusted sequences of the given table.
+      # Parameters:
+      # * +rep_prefix+:
+      #   The prefix put in front of all replication related database objects as
+      #   specified via Configuration#options.
+      #   Is used to create the sequences table.
+      # * +table_name+: name of the table
+      # Return value: a hash with
+      # * key: sequence name
+      # * value: a hash with
+      #   * :+increment+: current sequence increment
+      #   * :+value+: current value
+      def sequence_values(rep_prefix, table_name)
+        # check if the table has an auto_increment column, return if not
+        sequence_row = select_one(<<-end_sql)
+          show columns from #{table_name} where extra = 'auto_increment'
+        end_sql
+        return {} unless sequence_row
+        column_name = sequence_row['Field']
+
+        # check if the sequences table exists, create if necessary
+        sequence_table_name = "#{rep_prefix}_sequences"
+        unless tables.include?(sequence_table_name)
+          create_table "#{sequence_table_name}".to_sym,
+            :id => false, :options => 'ENGINE=MyISAM' do |t|
+            t.column :name, :string
+            t.column :current_value, :integer
+            t.column :increment, :integer
+            t.column :offset, :integer
+          end
+          ActiveRecord::Base.connection.execute(<<-end_sql) rescue nil
+            ALTER TABLE "#{sequence_table_name}"
+            ADD CONSTRAINT #{sequence_table_name}_pkey
+            PRIMARY KEY (name)
+          end_sql
+        end
+
+        sequence_row = select_one("select current_value, increment, offset from #{sequence_table_name} where name = '#{table_name}'")
+        if sequence_row == nil
+          current_max = select_one(<<-end_sql)['current_max'].to_i
+            select max(#{column_name}) as current_max from #{table_name}
+          end_sql
+          return {column_name => {
+              :increment => 1,
+              :value => current_max
+            }
+          }
+        else
+          return {column_name => {
+              :increment => sequence_row['increment'].to_i,
+              :value => sequence_row['offset'].to_i
+            }
+          }
+        end
+      end
+
+      # Ensures that the sequences of the named table (normally the primary key
+      # column) are generated with the correct increment and offset.
+      # * +rep_prefix+: not used (necessary) for the Postgres
+      # * +table_name+: name of the table (not used for Postgres)
+      # * +increment+: increment of the sequence
+      # * +offset+: offset
+      # * +left_sequence_values+:
+      #    hash as returned by #outdated_sequence_values for the left database
+      # * +right_sequence_values+:
+      #    hash as returned by #outdated_sequence_values for the right database
+      # * +adjustment_buffer+:
+      #    the "gap" that is created during sequence update to avoid concurrency problems
+      # E. g. an increment of 2 and offset of 1 will lead to generation of odd
+      # numbers.
+      def update_sequences(
+          rep_prefix, table_name, increment, offset,
+          left_sequence_values, right_sequence_values, adjustment_buffer)
+        return if left_sequence_values.empty?
+        column_name = left_sequence_values.keys[0]
+
+        # check if the sequences table exists, create if necessary
+        sequence_table_name = "#{rep_prefix}_sequences"
+        current_max =
+          [left_sequence_values[column_name][:value], right_sequence_values[column_name][:value]].max +
+          adjustment_buffer
+        new_start = current_max - (current_max % increment) + increment + offset
+
+        sequence_row = select_one("select current_value, increment, offset from #{sequence_table_name} where name = '#{table_name}'")
+        if sequence_row == nil
+          # no sequence exists yet for the table, create it and the according
+          # sequence trigger
+          execute(<<-end_sql)
+            insert into #{sequence_table_name}(name, current_value, increment, offset)
+            values('#{table_name}', #{new_start}, #{increment}, #{offset})
+          end_sql
+          trigger_name = "#{rep_prefix}_#{table_name}_sequence"
+          execute(<<-end_sql)
+            DROP TRIGGER IF EXISTS #{trigger_name};
+          end_sql
+
+          execute(<<-end_sql)
+            CREATE TRIGGER #{trigger_name}
+              BEFORE INSERT ON #{table_name} FOR EACH ROW BEGIN
+                IF NEW.#{column_name} = 0 THEN
+                  UPDATE #{sequence_table_name}
+                    SET current_value = LAST_INSERT_ID(current_value + increment)
+                    WHERE name = '#{table_name}';
+                  SET NEW.#{column_name} = LAST_INSERT_ID();
+                END IF;
+              END;
+          end_sql
+        elsif sequence_row['increment'].to_i != increment or sequence_row['offset'].to_i != offset
+          # sequence exists but with incorrect values; update it
+          execute(<<-end_sql)
+            update #{sequence_table_name}
+            set current_value = #{new_start},
+            increment = #{increment}, offset = #{offset}
+            where name = '#{table_name}'
+          end_sql
+        end
+      end
+
+      # Adds a big (8 byte value), auto-incrementing primary key column to the
+      # specified table.
+      # * table_name: name of the target table
+      # * key_name: name of the primary key column
+      def add_big_primary_key(table_name, key_name)
+        execute(<<-end_sql)
+          alter table #{table_name} add column #{key_name} bigint not null auto_increment primary key
+        end_sql
+      end
+
+      # Removes the custom sequence setup for the specified table.
+      # If no more rubyrep sequences are left, removes the sequence table.
+      # * +rep_prefix+: not used (necessary) for the Postgres
+      # * +table_name+: name of the table
+      def clear_sequence_setup(rep_prefix, table_name)
+        sequence_table_name = "#{rep_prefix}_sequences"
+        if tables.include?(sequence_table_name)
+          trigger_name = "#{rep_prefix}_#{table_name}_sequence"
+          trigger_row = select_one(<<-end_sql)
+            select * from information_schema.triggers
+            where trigger_schema = database()
+            and trigger_name = '#{trigger_name}'
+          end_sql
+          if trigger_row
+          execute "DROP TRIGGER #{trigger_name}"
+          execute "delete from #{sequence_table_name} where name = '#{table_name}'"
+          unless select_one("select * from #{sequence_table_name}")
+            # no more sequences left --> delete sequence table
+            drop_table sequence_table_name.to_sym
+          end
+        end
+      end
+    end
+  end
+end
+end
+

data/lib/rubyrep/replication_extenders/postgresql_replication.rb

@@ -0,0 +1,204 @@
+module RR
+  module ReplicationExtenders
+
+    # Provides PostgreSQL specific functionality for database replication
+    module PostgreSQLReplication
+      RR::ReplicationExtenders.register :postgresql => self
+
+      # Returns the key clause that is used in the trigger function.
+      # * +trigger_var+: should be either 'NEW' or 'OLD'
+      # * +params+: the parameter hash as described in #create_rep_trigger
+      def key_clause(trigger_var, params)
+        params[:keys].
+          map { |key| "'#{key}#{params[:key_sep]}' || #{trigger_var}.#{key}"}.
+          join(" || '#{params[:key_sep]}' || ")
+      end
+      private :key_clause
+
+      # Creates or replaces the replication trigger function.
+      # See #create_replication_trigger for a descriptions of the +params+ hash.
+      def create_or_replace_replication_trigger_function(params)
+        # first check, if PL/SQL is already activated and if not, do so.
+        if select_all("select lanname from pg_language where lanname = 'plpgsql'").empty?
+          execute "CREATE LANGUAGE plpgsql"
+        end
+
+        activity_check = ""
+        if params[:exclude_rr_activity] then
+          activity_check = <<-end_sql
+            PERFORM ACTIVE FROM #{params[:activity_table]};
+            IF FOUND THEN
+              RETURN NULL;
+            END IF;
+          end_sql
+        end
+
+        # now create the trigger
+        execute(<<-end_sql)
+          CREATE OR REPLACE FUNCTION #{params[:trigger_name]}() RETURNS TRIGGER AS $change_trigger$
+            BEGIN
+              #{activity_check}
+              IF (TG_OP = 'DELETE') THEN
+                INSERT INTO #{params[:log_table]}(change_table, change_key, change_type, change_time) 
+                  SELECT '#{params[:table]}', #{key_clause('OLD', params)}, 'D', now();
+              ELSIF (TG_OP = 'UPDATE') THEN
+                INSERT INTO #{params[:log_table]}(change_table, change_key, change_new_key, change_type, change_time)
+                  SELECT '#{params[:table]}', #{key_clause('OLD', params)}, #{key_clause('NEW', params)}, 'U', now();
+              ELSIF (TG_OP = 'INSERT') THEN
+                INSERT INTO #{params[:log_table]}(change_table, change_key, change_type, change_time)
+                  SELECT '#{params[:table]}', #{key_clause('NEW', params)}, 'I', now();
+              END IF;
+              RETURN NULL; -- result is ignored since this is an AFTER trigger
+            END;
+          $change_trigger$ LANGUAGE plpgsql;
+        end_sql
+
+      end
+
+      # Creates a trigger to log all changes for the given table.
+      # +params+ is a hash with all necessary information:
+      # * :+trigger_name+: name of the trigger
+      # * :+table+: name of the table that should be monitored
+      # * :+keys+: array of names of the key columns of the monitored table
+      # * :+log_table+: name of the table receiving all change notifications
+      # * :+activity_table+: name of the table receiving the rubyrep activity information
+      # * :+key_sep+: column seperator to be used in the key column of the log table
+      # * :+exclude_rr_activity+:
+      #   if true, the trigger will check and filter out changes initiated by RubyRep
+      def create_replication_trigger(params)
+        create_or_replace_replication_trigger_function params
+
+        execute(<<-end_sql)
+          CREATE TRIGGER #{params[:trigger_name]}
+          AFTER INSERT OR UPDATE OR DELETE ON #{params[:table]}
+              FOR EACH ROW EXECUTE PROCEDURE #{params[:trigger_name]}();
+        end_sql
+      end
+
+      # Removes a trigger and related trigger procedure.
+      # * +trigger_name+: name of the trigger
+      # * +table_name+: name of the table for which the trigger exists
+      def drop_replication_trigger(trigger_name, table_name)
+        execute "DROP TRIGGER #{trigger_name} ON #{table_name};"
+        execute "DROP FUNCTION #{trigger_name}();"
+      end
+
+      # Returns +true+ if the named trigger exists for the named table.
+      # * +trigger_name+: name of the trigger
+      # * +table_name+: name of the table
+      def replication_trigger_exists?(trigger_name, table_name)
+        search_path = select_one("show search_path")['search_path']
+        schemas = search_path.split(/,/).map { |p| quote(p) }.join(',')
+        !select_all(<<-end_sql).empty?
+          select 1 from information_schema.triggers
+          where event_object_schema in (#{schemas})
+          and trigger_name = '#{trigger_name}'
+          and event_object_table = '#{table_name}'
+        end_sql
+      end
+
+      # Returns all unadjusted sequences of the given table.
+      # Parameters:
+      # * +rep_prefix+: not used (necessary) for the Postgres
+      # * +table_name+: name of the table
+      # Return value: a hash with
+      # * key: sequence name
+      # * value: a hash with
+      #   * :+increment+: current sequence increment
+      #   * :+value+: current value
+      def sequence_values(rep_prefix, table_name)
+        result = {}
+        sequence_names = select_all(<<-end_sql).map { |row| row['relname'] }
+          select s.relname
+          from pg_class as t
+          join pg_depend as r on t.oid = r.refobjid
+          join pg_class as s on r.objid = s.oid
+          and s.relkind = 'S'
+          and t.relname = '#{table_name}'
+        end_sql
+        sequence_names.each do |sequence_name|
+          row = select_one("select last_value, increment_by from #{sequence_name}")
+          result[sequence_name] = {
+            :increment => row['increment_by'].to_i,
+            :value => row['last_value'].to_i
+          }
+        end
+        result
+      end
+
+      # Ensures that the sequences of the named table (normally the primary key
+      # column) are generated with the correct increment and offset.
+      # * +rep_prefix+: not used (necessary) for the Postgres
+      # * +table_name+: name of the table (not used for Postgres)
+      # * +increment+: increment of the sequence
+      # * +offset+: offset
+      # * +left_sequence_values+:
+      #    hash as returned by #outdated_sequence_values for the left database
+      # * +right_sequence_values+:
+      #    hash as returned by #outdated_sequence_values for the right database
+      # * +adjustment_buffer+:
+      #    the "gap" that is created during sequence update to avoid concurrency problems
+      # E. g. an increment of 2 and offset of 1 will lead to generation of odd
+      # numbers.
+      def update_sequences(
+          rep_prefix, table_name, increment, offset,
+          left_sequence_values, right_sequence_values, adjustment_buffer)
+        left_sequence_values.each do |sequence_name, left_current_value|
+          row = select_one("select last_value, increment_by from #{sequence_name}")
+          current_increment = row['increment_by'].to_i
+          current_value = row['last_value'].to_i
+          unless current_increment == increment and current_value % increment == offset
+            max_current_value =
+              [left_current_value[:value], right_sequence_values[sequence_name][:value]].max +
+              adjustment_buffer
+            new_start = max_current_value - (max_current_value % increment) + increment + offset
+            execute(<<-end_sql)
+            alter sequence "#{sequence_name}" increment by #{increment} restart with #{new_start}
+            end_sql
+          end
+        end
+      end
+
+      # Restores the original sequence settings.
+      # (Actually it sets the sequence increment to 1. If before, it had a
+      # different value, then the restoration will not be correct.)
+      # * +rep_prefix+: not used (necessary) for the Postgres
+      # * +table_name+: name of the table
+      def clear_sequence_setup(rep_prefix, table_name)
+        sequence_names = select_all(<<-end_sql).map { |row| row['relname'] }
+          select s.relname
+          from pg_class as t
+          join pg_depend as r on t.oid = r.refobjid
+          join pg_class as s on r.objid = s.oid
+          and s.relkind = 'S'
+          and t.relname = '#{table_name}'
+        end_sql
+        sequence_names.each do |sequence_name|
+          execute(<<-end_sql)
+              alter sequence "#{sequence_name}" increment by 1
+          end_sql
+        end
+      end
+
+      # Adds a big (8 byte value), auto-incrementing primary key column to the
+      # specified table.
+      # * table_name: name of the target table
+      # * key_name: name of the primary key column
+      def add_big_primary_key(table_name, key_name)
+        old_message_level = select_one("show client_min_messages")['client_min_messages']
+        execute "set client_min_messages = warning"
+        execute(<<-end_sql)
+          alter table "#{table_name}" add column #{key_name} bigserial
+        end_sql
+
+        execute(<<-end_sql)
+          alter table "#{table_name}" add constraint #{table_name}_#{key_name}_pkey primary key (#{key_name})
+        end_sql
+        
+      ensure
+        execute "set client_min_messages = #{old_message_level}"
+      end
+    end
+  end
+end
+

data/lib/rubyrep/replication_extenders/replication_extenders.rb

@@ -0,0 +1,26 @@
+module RR
+
+  # Replication extenders are modules that provide database specific functionality
+  # required for replication. They are mixed into ActiveRecord database connections.
+  # This module itself only provides functionality to register and retrieve
+  # such extenders.
+  module ReplicationExtenders
+    # Returns a Hash of currently registered replication extenders.
+    # (Empty Hash if no replication extenders were defined.)
+    def self.extenders
+      @extenders ||= {}
+      @extenders
+    end
+
+    # Registers one or multiple replication extender.
+    # extender is a Hash with
+    #   key::   The adapter symbol as used by ActiveRecord::Connection Adapters, e. g. :postgresql
+    #   value:: Name of the module implementing the replication extender
+    def self.register(extender)
+      @extenders ||= {}
+      @extenders.merge! extender
+    end
+  end
+end
+
+

data/lib/rubyrep/replication_helper.rb

@@ -0,0 +1,104 @@
+module RR
+
+  # Provides helper functionality for replicators.
+  # The methods exposed by this class are intended to provide a stable interface
+  # for third party replicators.
+  class ReplicationHelper
+
+    # The current +ReplicationRun+ instance
+    attr_accessor :replication_run
+
+    # The active +Session+
+    def session; replication_run.session; end
+
+    # Current options
+    def options; @options ||= session.configuration.options; end
+
+    # Delegates to Session#corresponding_table
+    def corresponding_table(db_arm, table); session.corresponding_table(db_arm, table); end
+
+    # Delegates to Committer#insert_record
+    def insert_record(database, table, values)
+      committer.insert_record(database, table, values)
+    end
+
+    # Delegates to Committer#insert_record
+    def update_record(database, table, values, old_key = nil)
+      committer.update_record(database, table, values, old_key)
+    end
+
+    # Delegates to Committer#insert_record
+    def delete_record(database, table, values)
+      committer.delete_record(database, table, values)
+    end
+
+    # Loads the specified record. Returns an according column_name => value hash.
+    # Parameters:
+    # * +database+: either :+left+ or :+right+
+    # * +table+: name of the table
+    # * +key+: A column_name => value hash for all primary key columns.
+    def load_record(database, table, key)
+      cursor = session.send(database).select_cursor(
+        :table => table,
+        :row_keys => [key],
+        :type_cast => true
+      )
+      row = nil
+      row = cursor.next_row if cursor.next?
+      cursor.clear
+      row
+    end
+
+    # The current Committer
+    attr_reader :committer
+    private :committer
+
+    # Asks the committer (if it exists) to finalize any open transactions
+    # +success+ should be true if there were no problems, false otherwise.
+    def finalize(success = true)
+      committer.finalize(success)
+    end
+
+    # Logs the outcome of a replication into the replication log table.
+    # * +diff+: the replicated ReplicationDifference
+    # * +outcome+: string summarizing the outcome of the replication
+    # * +details+: string with further details regarding the replication
+    def log_replication_outcome(diff, outcome, details = nil)
+      table = diff.changes[:left].table
+      key = diff.changes[:left].key
+      if key.size == 1
+        key = key.values[0]
+      else
+        key_parts = session.left.primary_key_names(table).map do |column_name|
+          %Q("#{column_name}"=>#{key[column_name].to_s.inspect})
+        end
+        key = key_parts.join(', ')
+      end
+      rep_details = details == nil ? nil : details[0...ReplicationInitializer::LONG_DESCRIPTION_SIZE]
+      diff_dump = diff.to_yaml[0...ReplicationInitializer::DIFF_DUMP_SIZE]
+      
+      session.left.insert_record "#{options[:rep_prefix]}_logged_events", {
+        :activity => 'replication',
+        :change_table => table,
+        :diff_type => diff.type.to_s,
+        :change_key => key,
+        :left_change_type => (diff.changes[:left] ? diff.changes[:left].type.to_s : nil),
+        :right_change_type => (diff.changes[:right] ? diff.changes[:right].type.to_s : nil),
+        :description => outcome.to_s,
+        :long_description => rep_details,
+        :event_time => Time.now,
+        :diff_dump => diff_dump
+      }
+    end
+    
+    # Creates a new SyncHelper for the given +TableSync+ instance.
+    def initialize(replication_run)
+      self.replication_run = replication_run
+
+      # Creates the committer. Important as it gives the committer the
+      # opportunity to start transactions
+      committer_class = Committers::committers[options[:committer]]
+      @committer = committer_class.new(session)
+    end
+  end
+end