rubyrep 1.0.0

data/lib/rubyrep/replication_initializer.rb

@@ -0,0 +1,307 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
+
+require 'rubyrep'
+
+module RR
+
+  # Ensures all preconditions are met to start with replication
+  class ReplicationInitializer
+
+    # The active Session
+    attr_accessor :session
+
+    # Creates a new RepInititializer for the given Session
+    def initialize(session)
+      self.session = session
+    end
+
+    # Returns the options for the given table.
+    # If table is +nil+, returns general options.
+    def options(table = nil)
+      if table
+        session.configuration.options_for_table table
+      else
+        session.configuration.options
+      end
+    end
+
+    # Creates a trigger logging all table changes
+    # * database: either :+left+ or :+right+
+    # * table: name of the table
+    def create_trigger(database, table)
+      options = self.options(table)
+
+      params = {
+        :trigger_name => "#{options[:rep_prefix]}_#{table}",
+        :table => table,
+        :keys => session.send(database).primary_key_names(table),
+        :log_table => "#{options[:rep_prefix]}_pending_changes",
+        :activity_table => "#{options[:rep_prefix]}_running_flags",
+        :key_sep => options[:key_sep],
+        :exclude_rr_activity => false,
+      }
+
+      session.send(database).create_replication_trigger params
+    end
+
+    # Returns +true+ if the replication trigger for the given table exists.
+    # * database: either :+left+ or :+right+
+    # * table: name of the table
+    def trigger_exists?(database, table)
+      trigger_name = "#{options(table)[:rep_prefix]}_#{table}"
+      session.send(database).replication_trigger_exists? trigger_name, table
+    end
+
+    # Drops the replication trigger of the named table.
+    # * database: either :+left+ or :+right+
+    # * table: name of the table
+    def drop_trigger(database, table)
+      trigger_name = "#{options(table)[:rep_prefix]}_#{table}"
+      session.send(database).drop_replication_trigger trigger_name, table
+    end
+
+    # Ensures that the sequences of the named table (normally the primary key
+    # column) are generated with the correct increment and offset in both
+    # left and right database.
+    # The sequence is always updated in both databases.
+    # * +table_pair+: a hash of names of corresponding :left and :right tables
+    # * +increment+: increment of the sequence
+    # * +left_offset+: offset of table in left database
+    # * +right_offset+: offset of table in right database
+    # E. g. an increment of 2 and offset of 1 will lead to generation of odd
+    # numbers.
+    def ensure_sequence_setup(table_pair, increment, left_offset, right_offset)
+      table_options = options(table_pair[:left])
+      if table_options[:adjust_sequences]
+        rep_prefix = table_options[:rep_prefix]
+        left_sequence_values = session.left.sequence_values rep_prefix, table_pair[:left]
+        right_sequence_values = session.right.sequence_values rep_prefix, table_pair[:right]
+        [:left, :right].each do |database|
+          offset = database == :left ? left_offset : right_offset
+          session.send(database).update_sequences \
+            rep_prefix, table_pair[database], increment, offset,
+            left_sequence_values, right_sequence_values, table_options[:sequence_adjustment_buffer]
+        end
+      end
+    end
+
+    # Restores the original sequence settings for the named table.
+    # (Actually it sets the sequence increment to 1. If before, it had a
+    # different value, then the restoration will not be correct.)
+    # * database: either :+left+ or :+right+
+    # * +table_name+: name of the table
+    def clear_sequence_setup(database, table)
+      table_options = options(table)
+      if table_options[:adjust_sequences]
+        session.send(database).clear_sequence_setup(
+          table_options[:rep_prefix], table
+        )
+      end
+    end
+
+    # Returns +true+ if the change log exists in the specified database.
+    # * database: either :+left+ or :+right+
+    def change_log_exists?(database)
+      session.send(database).tables.include? "#{options[:rep_prefix]}_pending_changes"
+    end
+
+    # Returns +true+ if the replication log exists.
+    def event_log_exists?
+      session.left.tables.include? "#{options[:rep_prefix]}_logged_events"
+    end
+
+    # Drops the change log table in the specified database
+    # * database: either :+left+ or :+right+
+    def drop_change_log(database)
+      session.send(database).drop_table "#{options[:rep_prefix]}_pending_changes"
+    end
+
+    # Drops the replication log table.
+    def drop_event_log
+      session.left.drop_table "#{options[:rep_prefix]}_logged_events"
+    end
+
+    # Size of the replication log column diff_dump
+    DIFF_DUMP_SIZE = 2000
+
+    # Size of the repliation log column long_description
+    LONG_DESCRIPTION_SIZE = 1000
+
+    # Ensures that create_table and related statements don't print notices to
+    # stdout. Then restored original message setting.
+    # * +database+: either :+left+ or :+right+
+    def silence_ddl_notices(database)
+      if session.configuration.send(database)[:adapter] =~ /postgres/
+        old_message_level = session.send(database).
+          select_one("show client_min_messages")['client_min_messages']
+        session.send(database).execute "set client_min_messages = warning"
+      end
+      yield
+    ensure
+      if session.configuration.send(database)[:adapter] =~ /postgres/
+        session.send(database).execute "set client_min_messages = #{old_message_level}"
+      end
+    end
+
+    # Creates the replication log table.
+    def create_event_log
+      silence_ddl_notices(:left) do
+        session.left.create_table "#{options[:rep_prefix]}_logged_events", :id => false do |t|
+          t.column :activity, :string
+          t.column :change_table, :string
+          t.column :diff_type, :string
+          t.column :change_key, :string
+          t.column :left_change_type, :string
+          t.column :right_change_type, :string
+          t.column :description, :string
+          t.column :long_description, :string, :limit => LONG_DESCRIPTION_SIZE
+          t.column :event_time, :timestamp
+          t.column :diff_dump, :string, :limit => DIFF_DUMP_SIZE
+        end
+        session.left.add_big_primary_key "#{options[:rep_prefix]}_logged_events", 'id'
+      end
+    end
+
+    # Creates the change log table in the specified database
+    # * database: either :+left+ or :+right+
+    def create_change_log(database)
+      silence_ddl_notices(database) do
+        session.send(database).create_table "#{options[:rep_prefix]}_pending_changes", :id => false do |t|
+          t.column :change_table, :string
+          t.column :change_key, :string
+          t.column :change_new_key, :string
+          t.column :change_type, :string
+          t.column :change_time, :timestamp
+        end
+        session.send(database).add_big_primary_key "#{options[:rep_prefix]}_pending_changes", 'id'
+      end
+    end
+
+    # Adds to the current session's configuration an exclusion of rubyrep tables.
+    def exclude_rubyrep_tables
+      session.configuration.exclude_rubyrep_tables
+    end
+
+    # Checks in both databases, if the activity marker tables exist and if not,
+    # creates them.
+    def ensure_activity_markers
+      table_name = "#{options[:rep_prefix]}_running_flags"
+      [:left, :right].each do |database|
+        unless session.send(database).tables.include? table_name
+          session.send(database).create_table table_name, :id => false do |t|
+            t.column :active, :integer
+          end
+        end
+      end
+    end
+
+    # Checks if the event log table already exists and creates it if necessary
+    def ensure_event_log
+      create_event_log unless event_log_exists?
+    end
+
+    # Checks in both databases, if the change log tables exists and creates them
+    # if necessary
+    def ensure_change_logs
+      [:left, :right].each do |database|
+        create_change_log(database) unless change_log_exists?(database)
+      end
+    end
+
+    # Checks in both databases, if the infrastructure tables (change log, event
+    # log) exist and creates them if necessary.
+    def ensure_infrastructure
+      ensure_activity_markers
+      ensure_change_logs
+      ensure_event_log
+    end
+
+    # Checks in both databases, if the change_log tables exist. If yes, drops them.
+    def drop_change_logs
+      [:left, :right].each do |database|
+        drop_change_log(database) if change_log_exists?(database)
+      end
+    end
+
+    # Checks in both databases, if the activity_marker tables exist. If yes, drops them.
+    def drop_activity_markers
+      table_name = "#{options[:rep_prefix]}_running_flags"
+      [:left, :right].each do |database|
+        if session.send(database).tables.include? table_name
+          session.send(database).drop_table table_name
+        end
+      end
+    end
+
+    # Removes all rubyrep infrastructure tables from both databases.
+    def drop_infrastructure
+      drop_event_log if event_log_exists?
+      drop_change_logs
+      drop_activity_markers
+    end
+
+    # Checks for tables that have triggers but are not in the list of configured
+    # tables. Removes triggers and restores sequences of those tables.
+    # * +configured_table_pairs+:
+    #   An array of table pairs (e. g. [{:left => 'xy', :right => 'xy2'}]).
+    def restore_unconfigured_tables(configured_table_pairs = session.configured_table_pairs)
+      [:left, :right].each do |database|
+        configured_tables = configured_table_pairs.map {|table_pair| table_pair[database]}
+        unconfigured_tables = session.send(database).tables - configured_tables
+        unconfigured_tables.each do |table|
+          if trigger_exists?(database, table)
+            drop_trigger(database, table)
+            session.send(database).execute(
+              "delete from #{options[:rep_prefix]}_pending_changes where change_table = '#{table}'")
+          end
+          clear_sequence_setup(database, table)
+        end
+      end
+    end
+
+    # Prepares the database / tables for replication.
+    def prepare_replication
+      exclude_rubyrep_tables
+
+      puts "Verifying RubyRep tables"
+      ensure_infrastructure
+
+      puts "Checking for and removing rubyrep triggers from unconfigured tables"
+      restore_unconfigured_tables
+
+      puts "Verifying rubyrep triggers of configured tables"
+      unsynced_table_pairs = []
+      table_pairs = session.sort_table_pairs(session.configured_table_pairs)
+      table_pairs.each do |table_pair|
+        table_options = options(table_pair[:left])
+        ensure_sequence_setup table_pair,
+          table_options[:sequence_increment],
+          table_options[:left_sequence_offset],
+          table_options[:right_sequence_offset]
+
+        unsynced = false
+        [:left, :right].each do |database|
+          unless trigger_exists? database, table_pair[database]
+            create_trigger database, table_pair[database]
+            unsynced = true
+          end
+        end
+        unsynced_table_pairs << table_pair if unsynced
+      end
+      unsynced_table_specs = unsynced_table_pairs.map do |table_pair|
+        "#{table_pair[:left]}, #{table_pair[:right]}"
+      end
+
+      unless unsynced_table_specs.empty?
+        puts "Executing initial table syncs"
+        runner = SyncRunner.new
+        runner.session = session
+        runner.options = {:table_specs => unsynced_table_specs}
+        runner.execute
+      end
+
+      puts "Starting replication"
+    end
+  end
+
+end

data/lib/rubyrep/replication_run.rb

@@ -0,0 +1,48 @@
+module RR
+  
+  # Executes a single replication run
+  class ReplicationRun
+
+    # The current Session object
+    attr_accessor :session
+
+    # Returns the current ReplicationHelper; creates it if necessary
+    def helper
+      @helper ||= ReplicationHelper.new(self)
+    end
+
+    # Returns the current replicator; creates it if necessary.
+    def replicator
+      @replicator ||=
+        Replicators.replicators[session.configuration.options[:replicator]].new(helper)
+    end
+
+    # Executes the replication run.
+    def run
+      success = false
+      replicator # ensure that replicator is created and has chance to validate settings
+
+      loop do
+        begin
+          session.reload_changes # ensure the cache of change log records is up-to-date
+          diff = ReplicationDifference.new session
+          diff.load
+          break unless diff.loaded?
+          replicator.replicate_difference diff if diff.type != :no_diff
+        rescue Exception => e
+          helper.log_replication_outcome diff, e.message, 
+            e.class.to_s + "\n" + e.backtrace.join("\n")
+        end
+      end
+      success = true # considered to be successful if we get till here
+    ensure
+      helper.finalize success
+    end
+
+    # Creates a new ReplicationRun instance.
+    # * +session+: the current Session
+    def initialize(session)
+      self.session = session
+    end
+  end
+end

data/lib/rubyrep/replication_runner.rb

@@ -0,0 +1,138 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'optparse'
+require 'thread'
+
+module RR
+  # This class implements the functionality of the 'replicate' command.
+  class ReplicationRunner
+
+    CommandRunner.register 'replicate' => {
+      :command => self,
+      :description => 'Starts a replication process'
+    }
+    
+    # Provided options. Possible values:
+    # * +:config_file+: path to config file
+    attr_accessor :options
+
+    # Should be set to +true+ if the replication runner should be terminated.
+    attr_accessor :termination_requested
+
+    # Parses the given command line parameter array.
+    # Returns the status (as per UNIX conventions: 1 if parameters were invalid,
+    # 0 otherwise)
+    def process_options(args)
+      status = 0
+      self.options = {}
+
+      parser = OptionParser.new do |opts|
+        opts.banner = <<EOS
+Usage: #{$0} replicate [options]
+
+  Replicates two databases as per specified configuration file.
+EOS
+        opts.separator ""
+        opts.separator "  Specific options:"
+
+        opts.on("-c", "--config", "=CONFIG_FILE",
+          "Mandatory. Path to configuration file.") do |arg|
+          options[:config_file] = arg
+        end
+
+        opts.on_tail("--help", "Show this message") do
+          $stderr.puts opts
+          self.options = nil
+        end
+      end
+
+      begin
+        parser.parse!(args)
+        if options # this will be +nil+ if the --help option is specified
+          raise("Please specify configuration file") unless options.include?(:config_file)
+        end
+      rescue Exception => e
+        $stderr.puts "Command line parsing failed: #{e}"
+        $stderr.puts parser.help
+        self.options = nil
+        status = 1
+      end
+
+      return status
+    end
+
+    # Returns the active +Session+.
+    # Loads config file and creates session if necessary.
+    def session
+      unless @session
+        load options[:config_file]
+        @session = Session.new Initializer.configuration
+      end
+      @session
+    end
+
+    # Wait for the next replication time
+    def pause_replication
+      @last_run ||= 1.year.ago
+      now = Time.now
+      @next_run = @last_run + session.configuration.options[:replication_interval]
+      unless now >= @next_run
+        waiting_time = @next_run - now
+        @waiter_thread.join waiting_time
+      end
+      @last_run = Time.now
+    end
+
+    # Initializes the waiter thread used for replication pauses and processing
+    # the process TERM signal.
+    def init_waiter
+      @termination_mutex = Mutex.new
+      @termination_mutex.lock
+      @waiter_thread ||= Thread.new {@termination_mutex.lock; self.termination_requested = true}
+      Signal.trap('TERM') {@termination_mutex.unlock}
+    end
+
+    # Prepares the replication
+    def prepare_replication
+      initializer = ReplicationInitializer.new session
+      initializer.prepare_replication
+    end
+
+    # Executes an endless loop of replication runs
+    def execute
+      init_waiter
+      prepare_replication
+
+      until termination_requested do
+        begin
+          session.refresh
+          run = ReplicationRun.new session
+          run.run
+        rescue Exception => e
+          now = Time.now.iso8601
+          $stderr.puts "#{now} Exception caught: #{e}"
+          if @last_exception_message != e.to_s # only print backtrace if something changed
+            @last_exception_message = e.to_s
+            $stderr.puts e.backtrace.map {|line| line.gsub(/^/, "#{' ' * now.length} ")}
+          end
+        end
+        pause_replication
+      end
+    end
+
+    # Entry points for executing a processing run.
+    # args: the array of command line options that were provided by the user.
+    def self.run(args)
+      runner = new
+
+      status = runner.process_options(args)
+      if runner.options
+        runner.execute
+      end
+      status
+    end
+
+  end
+end
+
+

data/lib/rubyrep/replicators/replicators.rb

@@ -0,0 +1,37 @@
+module RR
+  # Replicators are classes that implement the replication policies.
+  # This module provides functionality to register replicators and access the
+  # list of registered replicators.
+  # Each Replicator must register itself with Replicators#register.
+  # Each Replicator must implement the following methods:
+  #
+  #   # Creates a new replicator (A replicator is used for one replication run only)
+  #   #   * sync_helper: a SyncHelper object providing necessary information and functionalities
+  #   def initialize(sync_helper)
+  #
+  #   # Called to sync the provided difference.
+  #   # +difference+ is an instance of +ReplicationDifference+
+  #   def replicate_difference(difference)
+  #
+  #   # Provides default option for the replicator. Optional.
+  #   # Returns a hash with :key => value pairs.
+  #   def self.default_options
+  module Replicators
+    # Returns a Hash of currently registered replicators.
+    # (Empty Hash if no replicators were defined.)
+    def self.replicators
+      @replicators ||= {}
+      @replicators
+    end
+  
+    # Registers one or multiple replicators.
+    # syncer_hash is a Hash with
+    #   key::   The adapter symbol as used to reference the replicator
+    #   value:: The class implementing the replicator
+    def self.register(replicator_hash)
+      @replicators ||= {}
+      @replicators.merge! replicator_hash
+    end
+
+  end
+end