andyjeffries-rubyrep 1.2.1

data/lib/rubyrep/proxy_row_cursor.rb

@@ -0,0 +1,43 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
+
+require 'digest/sha1'
+
+require 'rubyrep'
+
+module RR
+  
+  # This class is used to scan a given table range 
+  # Can return rows either themselves or only their checksum
+  class ProxyRowCursor < ProxyCursor
+    
+    # The column_name => value hash of the current row.
+    attr_accessor :current_row
+    
+    # Creates a new cursor
+    #   * session: the current proxy session
+    #   * table: table_name
+    def initialize(session, table)
+      super
+    end
+    
+    # Returns true if there are unprocessed rows in the table range
+    def next?
+      cursor.next?
+    end
+    
+    # Returns the next row in cursor
+    def next_row
+      cursor.next_row
+    end
+    
+    # Returns for the next row
+    #   * a hash of :column_name => value pairs of the primary keys
+    #   * checksum string for that row
+    def next_row_keys_and_checksum
+      self.current_row = cursor.next_row
+      keys = self.current_row.reject {|key, | not primary_key_names.include? key}
+      checksum = Digest::SHA1.hexdigest(Marshal.dump(self.current_row))
+      return keys, checksum
+    end
+  end
+end

data/lib/rubyrep/proxy_runner.rb

@@ -0,0 +1,89 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'optparse'
+require 'drb'
+
+module RR
+  # This class implements the functionality of the rrproxy.rb command.
+  class ProxyRunner
+
+    CommandRunner.register 'proxy' => {
+      :command => self,
+      :description => 'Proxies connections from rubyrep commands to the database'
+    }
+    
+    # Default options to start a DatabaseProxy server
+    DEFAULT_OPTIONS = {
+      :port => DatabaseProxy::DEFAULT_PORT,
+      :host => ''
+    }
+    
+    # Parses the given command line parameter array.
+    # Returns 
+    #   * the options hash or nil if command line parsing failed
+    #   * status (as per UNIX conventions: 1 if parameters were invalid, 0 otherwise)
+    def get_options(args)
+      options = DEFAULT_OPTIONS
+      status = 0
+
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: #{$0} proxy [options]"
+        opts.separator ""
+        opts.separator "Specific options:"
+
+        opts.on("-h","--host", "=IP_ADDRESS", "IP address to listen on. Default: binds to all IP addresses of the computer") do |arg|
+          options[:host] = arg
+        end
+
+        opts.on("-p","--port", "=PORT_NUMBER", Integer, "TCP port to listen on. Default port: #{DatabaseProxy::DEFAULT_PORT}") do |arg|
+          options[:port] = arg
+        end
+
+        opts.on_tail("--help", "Show this message") do
+          $stderr.puts opts
+          options = nil
+        end
+      end
+
+      begin
+        parser.parse!(args)
+      rescue Exception => e
+        $stderr.puts "Command line parsing failed: #{e}"
+        $stderr.puts parser.help
+        options = nil
+        status = 1
+      end
+  
+      return options, status
+    end
+
+    # Builds the druby URL from the given options and returns it
+    def build_url(options)
+      "druby://#{options[:host]}:#{options[:port]}"
+    end
+
+    # Starts a proxy server under the given druby URL
+    def start_server(url)
+      proxy = DatabaseProxy.new
+
+      DRb.start_service(url, proxy)
+      DRb.thread.join
+    end
+    
+    # Runs the ProxyRunner (processing of command line & starting of server)
+    # args: the array of command line options with which to start the server
+    def self.run(args)
+      runner = ProxyRunner.new
+      
+      options, status = runner.get_options(args)
+      if options
+        url = runner.build_url(options)
+        runner.start_server(url)
+      end
+      status
+    end
+
+  end
+end
+
+

data/lib/rubyrep/replication_difference.rb

@@ -0,0 +1,100 @@
+module RR
+
+  # Describes a (record specific) difference between both databases as identifed
+  # via change log.
+  class ReplicationDifference
+
+    # The current Session.
+    def session
+      @session ||= loaders.session
+    end
+
+    # The current LoggedChangeLoaders instance
+    attr_accessor :loaders
+    
+    # The type of the difference. Either
+    # * :+left+: change in left database
+    # * :+right+: change in right database
+    # * :+conflict+: change in both databases
+    # * :+no_diff+: changes in both databases constitute no difference
+    attr_accessor :type
+
+    # Is set to +true+ if first replication attempt failed but it should be tried again later
+    attr_accessor :second_chance
+    alias_method :second_chance?, :second_chance
+
+    # A hash with keys :+left+ and / or :+right+.
+    # Hash values are LoggedChange instances.
+    def changes
+      @changes ||= {}
+    end
+
+    # Creates a new ReplicationDifference instance.
+    # +loaders+ is teh current LoggedChangeLoaders instance
+    def initialize(loaders)
+      self.loaders = loaders
+    end
+
+    # Should be set to +true+ if this ReplicationDifference instance was
+    # successfully loaded.
+    attr_writer :loaded
+
+    # Returns +true+ if a replication difference was loaded
+    def loaded?
+      @loaded
+    end
+
+    # Shortcut to calculate the "other" database.
+    OTHER_SIDE = {
+      :left => :right,
+      :right => :left
+    }
+
+    # Resulting diff type based on types of left changes (outer hash) and right
+    # changes (inner hash)
+    DIFF_TYPES = {
+      :insert =>    {:insert => :conflict, :update => :conflict, :delete => :conflict,  :no_change => :left},
+      :update =>    {:insert => :conflict, :update => :conflict, :delete => :conflict,  :no_change => :left},
+      :delete =>    {:insert => :conflict, :update => :conflict, :delete => :no_change, :no_change => :left},
+      :no_change => {:insert => :right,    :update => :right,    :delete => :right,     :no_change => :no_change}
+    }
+
+    # Amends a difference according to new entries in the change log table
+    def amend
+      loaders.update
+      changes[:left].load
+      changes[:right].load
+      self.type = DIFF_TYPES[changes[:left].type][changes[:right].type]
+    end
+    
+    # Loads a difference
+    def load
+      change_times = {}
+      [:left, :right].each do |database|
+        changes[database] = LoggedChange.new loaders[database]
+        change_times[database] = loaders[database].oldest_change_time
+      end
+      return if change_times[:left] == nil and change_times[:right] == nil
+
+      oldest = nil
+      [:left, :right].each do |database|
+        oldest = OTHER_SIDE[database] if change_times[database] == nil
+      end
+      oldest ||= change_times[:left] <= change_times[:right] ? :left : :right
+      changes[oldest].load_oldest
+
+      changes[OTHER_SIDE[oldest]].load_specified(
+        session.corresponding_table(oldest, changes[oldest].table),
+        changes[oldest].key)
+
+      self.type = DIFF_TYPES[changes[:left].type][changes[:right].type]
+      self.loaded = true
+    end
+
+    # Prevents session and change loaders from going into YAML output
+    def to_yaml_properties
+      instance_variables.sort.reject {|var_name| ['@session', '@loaders'].include? var_name}
+    end
+
+  end
+end

data/lib/rubyrep/replication_extenders/mysql_replication.rb

@@ -0,0 +1,271 @@
+module RR
+  module ReplicationExtenders
+
+    # Provides Mysql specific functionality for database replication
+    module MysqlReplication
+      RR::ReplicationExtenders.register :mysql2 => self
+
+      # Creates or replaces the replication trigger function.
+      # See #create_replication_trigger for a descriptions of the +params+ hash.
+      def create_or_replace_replication_trigger_function(params)
+        execute(<<-end_sql)
+          DROP PROCEDURE IF EXISTS `#{params[:trigger_name]}`;
+        end_sql
+        
+        activity_check = ""
+        if params[:exclude_rr_activity] then
+          activity_check = <<-end_sql
+            DECLARE active INT;
+            SELECT count(*) INTO active FROM #{params[:activity_table]};
+            IF active <> 0 THEN
+              LEAVE p;
+            END IF;
+          end_sql
+        end
+
+        execute(<<-end_sql)
+          CREATE PROCEDURE `#{params[:trigger_name]}`(change_key varchar(2000), change_new_key varchar(2000), change_type varchar(1))
+          p: BEGIN
+            #{activity_check}
+            INSERT INTO #{params[:log_table]}(change_table, change_key, change_new_key, change_type, change_time)
+              VALUES('#{params[:table]}', change_key, change_new_key, change_type, now());
+          END;
+        end_sql
+        
+      end
+
+      # Returns the key clause that is used in the trigger function.
+      # * +trigger_var+: should be either 'NEW' or 'OLD'
+      # * +params+: the parameter hash as described in #create_rep_trigger
+      def key_clause(trigger_var, params)
+        "concat_ws('#{params[:key_sep]}', " +
+          params[:keys].map { |key| "'#{key}', #{trigger_var}.#{quote_column_name(key)}"}.join(", ") +
+          ")"
+      end
+      private :key_clause
+
+      # Creates a trigger to log all changes for the given table.
+      # +params+ is a hash with all necessary information:
+      # * :+trigger_name+: name of the trigger
+      # * :+table+: name of the table that should be monitored
+      # * :+keys+: array of names of the key columns of the monitored table
+      # * :+log_table+: name of the table receiving all change notifications
+      # * :+activity_table+: name of the table receiving the rubyrep activity information
+      # * :+key_sep+: column seperator to be used in the key column of the log table
+      # * :+exclude_rr_activity+:
+      #   if true, the trigger will check and filter out changes initiated by RubyRep
+      def create_replication_trigger(params)
+        create_or_replace_replication_trigger_function params
+
+        %w(insert update delete).each do |action|
+          execute(<<-end_sql)
+            DROP TRIGGER IF EXISTS `#{params[:trigger_name]}_#{action}`;
+          end_sql
+
+          # The created triggers can handle the case where the trigger procedure
+          # is updated (that is: temporarily deleted and recreated) while the
+          # trigger is running.
+          # For that an MySQL internal exception is raised if the trigger
+          # procedure cannot be found. The exception is caught by an trigger
+          # internal handler. 
+          # The handler causes the trigger to retry calling the
+          # trigger procedure several times with short breaks in between.
+
+          trigger_var = action == 'delete' ? 'OLD' : 'NEW'
+          if action == 'update'
+            call_statement = "CALL `#{params[:trigger_name]}`(#{key_clause('OLD', params)}, #{key_clause('NEW', params)}, '#{action[0,1].upcase}');"
+          else
+            call_statement = "CALL `#{params[:trigger_name]}`(#{key_clause(trigger_var, params)}, null, '#{action[0,1].upcase}');"
+          end
+          execute(<<-end_sql)
+            CREATE TRIGGER `#{params[:trigger_name]}_#{action}`
+              AFTER #{action} ON `#{params[:table]}` FOR EACH ROW BEGIN
+                DECLARE number_attempts INT DEFAULT 0;
+                DECLARE failed INT;
+                DECLARE CONTINUE HANDLER FOR 1305 BEGIN
+                  DO SLEEP(0.05);
+                  SET failed = 1;
+                  SET number_attempts = number_attempts + 1;
+                END;
+                REPEAT
+                  SET failed = 0;
+                  #{call_statement}
+                UNTIL failed = 0 OR number_attempts >= 40 END REPEAT;
+              END;
+          end_sql
+        end
+
+      end
+
+      # Removes a trigger and related trigger procedure.
+      # * +trigger_name+: name of the trigger
+      # * +table_name+: name of the table for which the trigger exists
+      def drop_replication_trigger(trigger_name, table_name)
+        %w(insert update delete).each do |action|
+          execute "DROP TRIGGER `#{trigger_name}_#{action}`;"
+        end
+        execute "DROP PROCEDURE `#{trigger_name}`;"
+      end
+
+      # Returns +true+ if the named trigger exists for the named table.
+      # * +trigger_name+: name of the trigger
+      # * +table_name+: name of the table
+      def replication_trigger_exists?(trigger_name, table_name)
+        !select_all("select 1 from information_schema.triggers where trigger_schema = database() and trigger_name = '#{trigger_name}_insert' and event_object_table = '#{table_name}'").empty?
+      end
+
+      # Returns all unadjusted sequences of the given table.
+      # Parameters:
+      # * +rep_prefix+:
+      #   The prefix put in front of all replication related database objects as
+      #   specified via Configuration#options.
+      #   Is used to create the sequences table.
+      # * +table_name+: name of the table
+      # Return value: a hash with
+      # * key: sequence name
+      # * value: a hash with
+      #   * :+increment+: current sequence increment
+      #   * :+value+: current value
+      def sequence_values(rep_prefix, table_name)
+        # check if the table has an auto_increment column, return if not
+        sequence_row = select_one(<<-end_sql)
+          show columns from `#{table_name}` where extra = 'auto_increment'
+        end_sql
+        return {} unless sequence_row
+        column_name = sequence_row['Field']
+
+        # check if the sequences table exists, create if necessary
+        sequence_table_name = "#{rep_prefix}_sequences"
+        unless tables.include?(sequence_table_name)
+          create_table "#{sequence_table_name}".to_sym,
+            :id => false, :options => 'ENGINE=MyISAM' do |t|
+            t.column :name, :string
+            t.column :current_value, :integer
+            t.column :increment, :integer
+            t.column :offset, :integer
+          end
+          ActiveRecord::Base.connection.execute(<<-end_sql) rescue nil
+            ALTER TABLE "#{sequence_table_name}"
+            ADD CONSTRAINT #{sequence_table_name}_pkey
+            PRIMARY KEY (name)
+          end_sql
+        end
+
+        sequence_row = select_one("select current_value, increment, offset from #{sequence_table_name} where name = '#{table_name}'")
+        if sequence_row == nil
+          current_max = select_one(<<-end_sql)['current_max'].to_i
+            select max(`#{column_name}`) as current_max from `#{table_name}`
+          end_sql
+          return {column_name => {
+              :increment => 1,
+              :value => current_max
+            }
+          }
+        else
+          return {column_name => {
+              :increment => sequence_row['increment'].to_i,
+              :value => sequence_row['offset'].to_i
+            }
+          }
+        end
+      end
+
+      # Ensures that the sequences of the named table (normally the primary key
+      # column) are generated with the correct increment and offset.
+      # * +rep_prefix+: not used (necessary) for the Postgres
+      # * +table_name+: name of the table (not used for Postgres)
+      # * +increment+: increment of the sequence
+      # * +offset+: offset
+      # * +left_sequence_values+:
+      #    hash as returned by #sequence_values for the left database
+      # * +right_sequence_values+:
+      #    hash as returned by #sequence_values for the right database
+      # * +adjustment_buffer+:
+      #    the "gap" that is created during sequence update to avoid concurrency problems
+      # E. g. an increment of 2 and offset of 1 will lead to generation of odd
+      # numbers.
+      def update_sequences(
+          rep_prefix, table_name, increment, offset,
+          left_sequence_values, right_sequence_values, adjustment_buffer)
+        return if left_sequence_values.empty?
+        column_name = left_sequence_values.keys[0]
+
+        # check if the sequences table exists, create if necessary
+        sequence_table_name = "#{rep_prefix}_sequences"
+        current_max =
+          [left_sequence_values[column_name][:value], right_sequence_values[column_name][:value]].max +
+          adjustment_buffer
+        new_start = current_max - (current_max % increment) + increment + offset
+
+        sequence_row = select_one("select current_value, increment, offset from #{sequence_table_name} where name = '#{table_name}'")
+        if sequence_row == nil
+          # no sequence exists yet for the table, create it and the according
+          # sequence trigger
+          execute(<<-end_sql)
+            insert into #{sequence_table_name}(name, current_value, increment, offset)
+            values('#{table_name}', #{new_start}, #{increment}, #{offset})
+          end_sql
+          trigger_name = "#{rep_prefix}_#{table_name}_sequence"
+          execute(<<-end_sql)
+            DROP TRIGGER IF EXISTS `#{trigger_name}`;
+          end_sql
+
+          execute(<<-end_sql)
+            CREATE TRIGGER `#{trigger_name}`
+              BEFORE INSERT ON `#{table_name}` FOR EACH ROW BEGIN
+                IF NEW.`#{column_name}` = 0 THEN
+                  UPDATE #{sequence_table_name}
+                    SET current_value = LAST_INSERT_ID(current_value + increment)
+                    WHERE name = '#{table_name}';
+                  SET NEW.`#{column_name}` = LAST_INSERT_ID();
+                END IF;
+              END;
+          end_sql
+        elsif sequence_row['increment'].to_i != increment or sequence_row['offset'].to_i != offset
+          # sequence exists but with incorrect values; update it
+          execute(<<-end_sql)
+            update #{sequence_table_name}
+            set current_value = #{new_start},
+            increment = #{increment}, offset = #{offset}
+            where name = '#{table_name}'
+          end_sql
+        end
+      end
+
+      # Adds a big (8 byte value), auto-incrementing primary key column to the
+      # specified table.
+      # * table_name: name of the target table
+      # * key_name: name of the primary key column
+      def add_big_primary_key(table_name, key_name)
+        execute(<<-end_sql)
+          alter table #{table_name} add column #{key_name} bigint not null auto_increment primary key
+        end_sql
+      end
+
+      # Removes the custom sequence setup for the specified table.
+      # If no more rubyrep sequences are left, removes the sequence table.
+      # * +rep_prefix+: not used (necessary) for the Postgres
+      # * +table_name+: name of the table
+      def clear_sequence_setup(rep_prefix, table_name)
+        sequence_table_name = "#{rep_prefix}_sequences"
+        if tables.include?(sequence_table_name)
+          trigger_name = "#{rep_prefix}_#{table_name}_sequence"
+          trigger_row = select_one(<<-end_sql)
+            select * from information_schema.triggers
+            where trigger_schema = database()
+            and trigger_name = '#{trigger_name}'
+          end_sql
+          if trigger_row
+          execute "DROP TRIGGER `#{trigger_name}`"
+          execute "delete from #{sequence_table_name} where name = '#{table_name}'"
+          unless select_one("select * from #{sequence_table_name}")
+            # no more sequences left --> delete sequence table
+            drop_table sequence_table_name.to_sym
+          end
+        end
+      end
+    end
+  end
+end
+end
+