rubyrep 1.0.0

data/lib/rubyrep/replicators/two_way_replicator.rb

@@ -0,0 +1,334 @@
+module RR
+  module Replicators
+    # This replicator implements a two way replication.
+    # Options:
+    # * :+left_change_handling+, :+right_change_handling+:
+    #   Handling of records that were changed only in the named database.
+    #   Can be any of the following:
+    #   * :+ignore+: No action.
+    #   * :+replicate+: Updates other database accordingly. *Default* *Setting*
+    #   * +Proc+ object:
+    #     If a Proc object is given, it is responsible for dealing with the
+    #     record. Called with the following parameters:
+    #     * replication_helper: The current ReplicationHelper instance.
+    #     * difference: A ReplicationDifference instance describing the change
+    # * :+replication_conflict_handling+:
+    #   Handling of conflicting record changes. Can be any of the following:
+    #   * :+ignore+: No action. *Default* *Setting*
+    #   * :+left_wins+: The right database is updated accordingly.
+    #   * :+right_wins+: The left database is updated accordingly.
+    #   * :+later_wins+:
+    #     The more recent change is replicated.
+    #     (If both changes have same age: left change is replicated)
+    #   * :+earlier_wins+:
+    #     The less recent change is replicated.
+    #     (If both records have same age: left change is replicated)
+    #   * +Proc+ object:
+    #     If a Proc object is given, it is responsible for dealing with the
+    #     record. Called with the following parameters:
+    #     * replication_helper: The current ReplicationHelper instance.
+    #     * difference: A ReplicationDifference instance describing the changes
+    # * :+logged_replication_events+:
+    #   Specifies which types of replications are logged.
+    #   Is either a single value or an array of multiple ones.
+    #   Default: [:ignored_conflicts]
+    #   Possible values:
+    #   * :+ignored_changes+: log ignored (but not replicated) non-conflict changes
+    #   * :+all_changes+: log all non-conflict changes
+    #   * :+ignored_conflicts+: log ignored (but not replicated) conflicts
+    #   * :+all_conflicts+: log all conflicts
+    #
+    # Example of using a Proc object for custom behaviour:
+    #   lambda do |rep_helper, diff|
+    #     # if specified as replication_conflict_handling option, logs all
+    #     # conflicts to a text file
+    #     File.open('/var/log/rubyrep_conflict_log', 'a') do |f|
+    #       f.puts <<-end_str
+    #         #{Time.now}: conflict
+    #         * in table #{diff.changes[:left].table}
+    #         * for record '#{diff.changes[:left].key}'
+    #         * change type in left db: '#{diff.changes[:left].type}'
+    #         * change type in right db: '#{diff.changes[:right].type}'
+    #       end_str
+    #     end
+    #   end
+    class TwoWayReplicator
+      
+      # Register the syncer
+      Replicators.register :two_way => self
+
+      # The current ReplicationHelper object
+      attr_accessor :rep_helper
+
+      # Default TwoWayReplicator options.
+      DEFAULT_OPTIONS =  {
+        :left_change_handling => :replicate,
+        :right_change_handling => :replicate,
+        :replication_conflict_handling => :ignore,
+        :logged_replication_events => [:ignored_conflicts],
+      }
+
+      # Checks if an option is configured correctly. Raises an ArgumentError if not.
+      # * +table_spec+: the table specification to which the option belongs. May be +nil+.
+      # * +valid_option_values+: array of valid option values
+      # * +option_key+: the key of the option that is to be checked
+      # * +option_value+: the value of the option that is to be checked
+      def verify_option(table_spec, valid_option_values, option_key, option_value)
+        unless valid_option_values.include? option_value
+          message = ""
+          message << "#{table_spec.inspect}: " if table_spec
+          message << "#{option_value.inspect} not a valid #{option_key.inspect} option"
+          raise ArgumentError.new(message)
+        end
+      end
+
+      # Verifies if the :+left_change_handling+ / :+right_change_handling+
+      # options are valid.
+      # Raises an ArgumentError if an option is invalid
+      def validate_change_handling_options
+        [:left_change_handling, :right_change_handling].each do |key|
+          rep_helper.session.configuration.each_matching_option(key) do |table_spec, value|
+            unless value.respond_to? :call
+              verify_option table_spec, [:ignore, :replicate], key, value
+            end
+          end
+        end
+      end
+
+      # Verifies if the given :+replication_conflict_handling+ options are valid.
+      # Raises an ArgumentError if an option is invalid.
+      def validate_conflict_handling_options
+        rep_helper.session.configuration.each_matching_option(:replication_conflict_handling) do |table_spec, value|
+          unless value.respond_to? :call
+            verify_option table_spec,
+              [:ignore, :left_wins, :right_wins, :later_wins, :earlier_wins],
+              :replication_conflict_handling, value
+          end
+        end
+      end
+
+      # Verifies if the given :+replication_logging+ option /options is / are valid.
+      # Raises an ArgumentError if invalid
+      def validate_logging_options
+        rep_helper.session.configuration.each_matching_option(:logged_replication_events) do |table_spec, values|
+          values = [values].flatten # ensure that I have an array
+          values.each do |value|
+            verify_option table_spec,
+              [:ignored_changes, :all_changes, :ignored_conflicts, :all_conflicts],
+              :logged_replication_events, value
+          end
+        end
+      end
+
+      # Initializes the TwoWayReplicator
+      # Raises an ArgumentError if any of the replication options is invalid.
+      #
+      # Parameters:
+      # * rep_helper:
+      #   The ReplicationHelper object providing information and utility functions.
+      def initialize(rep_helper)
+        self.rep_helper = rep_helper
+
+        validate_change_handling_options
+        validate_conflict_handling_options
+        validate_logging_options
+      end
+
+      # Shortcut to calculate the "other" database.
+      OTHER_SIDE = {
+        :left => :right,
+        :right => :left
+      }
+
+      # Specifies how to clear conflicts.
+      # The outer hash keys describe the type of the winning change.
+      # The inner hash keys describe the type of the loosing change.
+      # The inser hash values describe the action to take on the loosing side.
+      CONFLICT_STATE_MATRIX = {
+        :insert => {:insert => :update, :update => :update, :delete => :insert},
+        :update => {:insert => :update, :update => :update, :delete => :insert},
+        :delete => {:insert => :delete, :update => :delete, :delete => :delete}
+      }
+
+      # Helper function that clears a conflict by taking the change from the
+      # specified winning database and updating the other database accordingly.
+      # * +source_db+: the winning database (either :+left+ or :+right+)
+      # * +diff+: the ReplicationDifference instance
+      # * +remaining_attempts+: the number of remaining replication attempts for this difference
+      def clear_conflict(source_db, diff, remaining_attempts)
+        source_change = diff.changes[source_db]
+        target_db = OTHER_SIDE[source_db]
+        target_change = diff.changes[target_db]
+
+        target_action = CONFLICT_STATE_MATRIX[source_change.type][target_change.type]
+        source_key = source_change.type == :update ? source_change.new_key : source_change.key
+        target_key = target_change.type == :update ? target_change.new_key : target_change.key
+        case target_action
+        when :insert
+          attempt_insert source_db, diff, remaining_attempts, source_key
+        when :update
+          attempt_update source_db, diff, remaining_attempts, source_key, target_key
+        when :delete
+          attempt_delete source_db, diff, target_key
+        end
+      end
+
+      # Returns the options for the specified table name.
+      # * +table+: name of the table (left database version)
+      def options_for_table(table)
+        @options_for_table ||= {}
+        unless @options_for_table.include? table
+          @options_for_table[table] = DEFAULT_OPTIONS.merge(
+            rep_helper.session.configuration.options_for_table(table))
+        end
+        @options_for_table[table]
+      end
+
+      # Logs replication of the specified difference as per configured
+      # :+replication_conflict_logging+ / :+left_change_logging+ / :+right_change_logging+ options.
+      # * +winner+: Either the winner database (:+left+ or :+right+) or :+ignore+
+      # * +diff+: the ReplicationDifference instance
+      def log_replication_outcome(winner, diff)
+        options = options_for_table(diff.changes[:left].table)
+        option_values = [options[:logged_replication_events]].flatten # make sure I have an array
+        if diff.type == :conflict
+          return unless option_values.include?(:all_conflicts) or option_values.include?(:ignored_conflicts)
+          return if winner != :ignore and not option_values.include?(:all_conflicts)
+          outcome = {:left => 'left_won', :right => 'right_won', :ignore => 'ignored'}[winner]
+        else
+          return unless option_values.include?(:all_changes) or option_values.include?(:ignored_changes)
+          return if winner != :ignore and not option_values.include?(:all_changes)
+          outcome = winner == :ignore ? 'ignored' : 'replicated'
+        end
+        rep_helper.log_replication_outcome diff, outcome
+      end
+
+      # How often a replication will be attempted (in case it fails because the
+      # record in question was removed from the source or inserted into the
+      # target database _after_ the ReplicationDifference was loaded
+      MAX_REPLICATION_ATTEMPTS = 2
+
+      # Attempts to read the specified record from the source database and insert
+      # it into the target database.
+      # Retries if insert fails due to missing source or suddenly existing target
+      # record.
+      # * +source_db+: either :+left+ or :+right+ - source database of replication
+      # * +diff+: the current ReplicationDifference instance
+      # * +remaining_attempts+: the number of remaining replication attempts for this difference
+      # * +source_key+: a column_name => value hash identifying the source record
+      def attempt_insert(source_db, diff, remaining_attempts, source_key)
+        source_change = diff.changes[source_db]
+        source_table = source_change.table
+        target_db = OTHER_SIDE[source_db]
+        target_table = rep_helper.corresponding_table(source_db, source_table)
+
+        values = rep_helper.load_record source_db, source_table, source_key
+        if values == nil
+          diff.amend
+          replicate_difference diff, remaining_attempts - 1
+        else
+          begin
+            # note: savepoints have to be used for postgresql (as a failed SQL
+            #       statement will otherwise invalidate the complete transaction.)
+            rep_helper.session.send(target_db).execute "savepoint rr_insert"
+            log_replication_outcome source_db, diff
+            rep_helper.insert_record target_db, target_table, values
+          rescue Exception => e
+            rep_helper.session.send(target_db).execute "rollback to savepoint rr_insert"
+            row = rep_helper.load_record target_db, target_table, source_key
+            raise unless row # problem is not the existence of the record in the target db
+            diff.amend
+            replicate_difference diff, remaining_attempts - 1
+          end
+        end
+      end
+
+      # Attempts to read the specified record from the source database and update
+      # the specified record in the target database.
+      # Retries if update fails due to missing source
+      # * +source_db+: either :+left+ or :+right+ - source database of replication
+      # * +diff+: the current ReplicationDifference instance
+      # * +remaining_attempts+: the number of remaining replication attempts for this difference
+      # * +source_key+: a column_name => value hash identifying the source record
+      # * +target_key+: a column_name => value hash identifying the source record
+      def attempt_update(source_db, diff, remaining_attempts, source_key, target_key)
+        source_change = diff.changes[source_db]
+        source_table = source_change.table
+        target_db = OTHER_SIDE[source_db]
+        target_table = rep_helper.corresponding_table(source_db, source_table)
+
+        values = rep_helper.load_record source_db, source_table, source_key
+        if values == nil
+          diff.amend
+          replicate_difference diff, remaining_attempts - 1
+        else
+          log_replication_outcome source_db, diff
+          rep_helper.update_record target_db, target_table, values, target_key
+        end
+      end
+
+      # Attempts delete the source record from the target database.
+      # E. g. if +source_db is :+left+, then the record is deleted in database
+      # :+right+.
+      # * +source_db+: either :+left+ or :+right+ - source database of replication
+      # * +diff+: the current ReplicationDifference instance
+      # * +target_key+: a column_name => value hash identifying the source record
+      def attempt_delete(source_db, diff, target_key)
+        change = diff.changes[source_db]
+        target_db = OTHER_SIDE[source_db]
+        target_table = rep_helper.corresponding_table(source_db, change.table)
+        log_replication_outcome source_db, diff
+        rep_helper.delete_record target_db, target_table, target_key
+      end
+
+      # Called to replicate the specified difference.
+      # * :+diff+: ReplicationDifference instance
+      # * :+remaining_attempts+: how many more times a replication will be attempted
+      def replicate_difference(diff, remaining_attempts = MAX_REPLICATION_ATTEMPTS)
+        raise Exception, "max replication attempts exceeded" if remaining_attempts == 0
+        options = options_for_table(diff.changes[:left].table)
+        if diff.type == :left or diff.type == :right
+          key = diff.type == :left ? :left_change_handling : :right_change_handling
+          option = options[key]
+
+          if option == :ignore
+            log_replication_outcome :ignore, diff
+          elsif option == :replicate
+            source_db = diff.type
+
+            change = diff.changes[source_db]
+
+            case change.type
+            when :insert
+              attempt_insert source_db, diff, remaining_attempts, change.key
+            when :update
+              attempt_update source_db, diff, remaining_attempts, change.new_key, change.key
+            when :delete
+              attempt_delete source_db, diff, change.key
+            end
+          else # option must be a Proc
+            option.call rep_helper, diff
+          end
+        elsif diff.type == :conflict
+          option = options[:replication_conflict_handling]
+          if option == :ignore
+            log_replication_outcome :ignore, diff
+          elsif option == :left_wins
+            clear_conflict :left, diff, remaining_attempts
+          elsif option == :right_wins
+            clear_conflict :right, diff, remaining_attempts
+          elsif option == :later_wins
+            winner_db = diff.changes[:left].last_changed_at >= diff.changes[:right].last_changed_at ? :left : :right
+            clear_conflict winner_db, diff, remaining_attempts
+          elsif option == :earlier_wins
+            winner_db = diff.changes[:left].last_changed_at <= diff.changes[:right].last_changed_at ? :left : :right
+            clear_conflict winner_db, diff, remaining_attempts
+          else # option must be a Proc
+            option.call rep_helper, diff
+          end
+        end
+      end
+      
+    end
+  end
+end

data/lib/rubyrep/scan_progress_printers/progress_bar.rb

@@ -0,0 +1,65 @@
+module RR
+  module ScanProgressPrinters
+    
+    # A helper class to print a text progress bar.
+    class ProgressBar
+
+      MAX_MARKERS = 25 #length of the progress bar (in characters)
+
+      # Register ProgressBar with the given command line options.
+      # (Command line format as specified by OptionParser#on.)
+      # First argument is the key through which the printer can be refered in
+      # the configuration file
+      RR::ScanProgressPrinters.register :progress_bar, self,
+        "-b", "--progress-bar[=length]",
+        "Show the progress of the table scanning process as progress bar."
+
+      # Receives the command line argument
+      cattr_accessor :arg
+
+      # Returns the length (in characters) of the progress bar.
+      def max_markers
+        @max_markers ||= arg ? arg.to_i : MAX_MARKERS
+      end
+
+      # Creates a new progress bar.
+      # * +max_steps+: number of steps at completion
+      # * +session+: the current Session
+      # * +left_table+: name of the left database table
+      # * +right_table+: name of the right database table
+      def initialize(max_steps, session, left_table, right_table)
+        @use_ansi = session.configuration.options_for_table(left_table)[:use_ansi]
+        @max_steps, @current_steps = max_steps, 0
+        @steps_per_marker = @max_steps.to_f / max_markers
+        @current_markers, @current_percentage = 0, 0
+      end
+  
+      # Increases progress by +step_increment+ steps.
+      def step(step_increment = 1)
+        @current_steps+= step_increment
+        new_markers = @max_steps != 0 ? (@current_steps / @steps_per_marker).to_i : max_markers
+
+        new_percentage = @max_steps != 0 ? @current_steps * 100 / @max_steps : 100
+        if @use_ansi and new_percentage != @current_percentage
+          # This part uses ANSI escape sequences to show a running percentage
+          # to the left of the progress bar
+          print "\e[1D" * (@current_markers + 5) if @current_percentage != 0 # go left
+          print "#{new_percentage}%".rjust(4) << " "
+          print "\e[1C" * @current_markers if @current_markers != 0 # go back right
+          $stdout.flush
+          @current_percentage = new_percentage
+        end
+
+        if new_markers > @current_markers
+          print '.'  * (new_markers - @current_markers)
+          @current_markers = new_markers
+          $stdout.flush
+        end
+        if @current_steps == @max_steps
+          print '.' * (max_markers - @current_markers) + ' '
+          $stdout.flush
+        end
+      end
+    end
+  end
+end

data/lib/rubyrep/scan_progress_printers/scan_progress_printers.rb

@@ -0,0 +1,65 @@
+module RR
+  # Manages scan progress printers. Scan progress printers implement functionality
+  # to report the progress of a table scan.
+  # *Each* table scan is handled by a *separate* printer instance.
+  #
+  # Scan progress printers need to register themselves and their command line options
+  # with #register.
+  #
+  # A scan progress printer needs to implement at the minimum the following
+  # functionality:
+  #
+  #   # Receives the command line argument as yielded by OptionParser#on.
+  #   def self.arg=(arg)
+  #
+  #   # Creation of a new ScanProgressPrinter.
+  #   # * +max_steps+: number of steps at completion
+  #   # * +session+: the current Session
+  #   # * +left_table+: name of the left database table
+  #   # * +right_table+: name of the right database table
+  #   def initialize(max_steps, left_table, right_table)
+  #
+  #   # Progress is advanced by +progress+ number of steps.
+  #   def step(progress)
+  #
+  module ScanProgressPrinters
+
+    # Hash of registered ScanProgressPrinters.
+    # Each entry is a hash with the following key and related value:
+    # * key: Identifier of the progress printer
+    # * value: A hash with payload information. Possible values:
+    #   * :+printer_class+: The ScanProgressPrinter class.
+    #   * :+opts+: An array defining the command line options (handed to OptionParter#on).
+    def self.printers
+      @@progress_printers ||= {}
+    end
+
+    # Needs to be called by ScanProgressPrinters to register themselves (+printer+)
+    # and their command line options.
+    # * :+printer_id+ is the symbol through which the printer can be referenced.
+    # * :+printer_class+ is the ScanProgressPrinter class,
+    # * :+opts+ is an array defining the command line options (handed to OptionParter#on).
+    def self.register(printer_id, printer_class, *opts)
+      printers[printer_id] = {
+        :printer_class => printer_class,
+        :opts => opts
+      }
+    end
+
+    # Registers all report printer command line options into the given
+    # OptionParser.
+    # Once the command line is parsed with OptionParser#parse! it will
+    # yield the correct printer class.
+    #
+    # Note:
+    # If multiple printers are specified in the command line, all are yielded.
+    def self.on_printer_selection(opts)
+      printers.each_value do |printer|
+        opts.on(*printer[:opts]) do |arg|
+          printer[:printer_class].arg = arg
+          yield printer[:printer_class]
+        end
+      end
+    end
+  end
+end

data/lib/rubyrep/scan_report_printers/scan_detail_reporter.rb

@@ -0,0 +1,111 @@
+require 'fileutils'
+require 'tmpdir'
+require 'tempfile'
+require 'yaml'
+
+module RR::ScanReportPrinters
+  # A ScanReportPrinter producing a summary (number of differences) only.
+  class ScanDetailReporter < ScanSummaryReporter
+    
+    # Register ScanSummaryReporter with the given command line options.
+    # (Command line format as specified by OptionParser#on.)
+    RR::ScanReportPrinters.register self, "-d", "--detailed[=mode]",
+      "Print the number of differences of each table. E. g.",
+      "  left_table / right_table [differences]",
+      "followed by a full dump of the differences in YAML format.",
+      "The 'mode' argument determines how the row differences are printed:",
+      " * full shows the full records",
+      " * keys shows the primary key columns only",
+      " * diff shows the primary key and differing columsn only"
+    
+    # The current Session object
+    attr_accessor :session
+
+    # The temporary File receiving the differences
+    attr_accessor :tmpfile
+
+    # Mode of reporting. Should be either
+    # * :+full+
+    # * :+keys+ or
+    # * :+diff+
+    attr_accessor :report_mode
+
+    # Array of names of the primary key columns of the table currently being
+    # scanned.
+    attr_accessor :primary_key_names
+
+    # A scan run is to be started using this scan result printer.
+    # +arg+ is the command line argument as yielded by OptionParser#on.
+    def initialize(session, arg)
+      super session, ""
+      self.session = session
+
+      self.report_mode = case arg
+      when 'diff'
+        :diff
+      when 'keys'
+        :keys
+      else
+        :full
+      end
+    end
+    
+    # A scan of the given 'left' table and corresponding 'right' table is executed.
+    # Needs to yield so that the actual scan can be executed.
+    def scan(left_table, right_table)
+
+      super left_table, right_table
+
+    ensure
+      self.primary_key_names = nil
+      if self.tmpfile
+        self.tmpfile.close
+        self.tmpfile.open
+        self.tmpfile.each_line {|line| puts line}
+        self.tmpfile.close!
+        self.tmpfile = nil
+      end
+    end
+
+    # Returns a cleaned row as per current +report_mode+.
+    # +row+ is either a column_name => value hash or an array of 2 such rows.
+    def clear_columns(row)
+      case report_mode
+      when :full
+        row
+      when :keys
+        row = row[0] if row.kind_of?(Array)
+        self.primary_key_names ||= session.left.primary_key_names(self.left_table)
+        row.reject {|column, value| !self.primary_key_names.include?(column)}
+      when :diff
+        self.primary_key_names ||= session.left.primary_key_names(self.left_table)
+        if row.kind_of?(Array)
+          new_row_array = [{}, {}]
+          row[0].each do |column, value|
+            if self.primary_key_names.include?(column) or value != row[1][column]
+              new_row_array[0][column] = row[0][column]
+              new_row_array[1][column] = row[1][column]
+            end
+          end
+          new_row_array
+        else
+          row
+        end
+      end
+    end
+
+    # Each difference is handed to the printer as described in the format
+    # as described e. g. in DirectTableScan#run
+    def report_difference(type, row)
+      self.tmpfile ||= Tempfile.new 'rubyrep_scan_details'
+      tmpfile.puts({type => clear_columns(row)}.to_yaml)
+      super type, row
+    end
+
+    # Optional method. If a scan report printer has it, it is called after the
+    # last table scan is executed.
+    # (A good place to print a final summary.)
+    def scanning_finished
+    end
+  end
+end

data/lib/rubyrep/scan_report_printers/scan_report_printers.rb

@@ -0,0 +1,67 @@
+module RR
+  # Manages scan report printers. Scan report printers implement functionality
+  # to report the row differences identified during a scan.
+  # *All* table scans are processed by the *same* printer instance.
+  #
+  # Scan report printers need to register themselves and their command line options
+  # with #register.
+  #
+  # A scan report printer needs to implement at the minimum the following
+  # functionality:
+  #
+  #   # Creation of a new ScanReportPrinter.
+  #   # * +session+: the current Session object
+  #   # * +arg+: command line argument as yielded by OptionParser#on.
+  #   def initialize(arg)
+  #
+  #   # A scan of the given 'left' table and corresponding 'right' table is executed.
+  #   # Needs to yield so that the actual scan can be executed.
+  #   def scan(left_table, right_table)
+  #
+  #   # Each difference is handed to the printer as described in the format
+  #   # as described e. g. in DirectTableScan#run
+  #   def report_difference(type, row)
+  #
+  #   # Optional method. If a scan report printer has it, it is called after the
+  #   # last table scan is executed.
+  #   # (A good place to print a final summary.)
+  #   def scanning_finished
+  #
+  module ScanReportPrinters
+
+    # Array of registered ScanReportPrinters.
+    # Each entry is a hash with the following keys and related values:
+    # * :+printer_class+: The ScanReportPrinter class.
+    # * :+opts+: An array defining the command line options (handed to OptionParter#on).
+    def self.printers
+      @@report_printers ||= []
+    end
+
+    # Needs to be called by ScanReportPrinters to register themselves (+printer+)
+    # and their command line options.
+    # * :+printer_class+ is the ScanReportPrinter class,
+    # * :+opts+ is an array defining the command line options (handed to OptionParter#on).
+    def self.register(printer_class, *opts)
+      printers << {
+        :printer_class => printer_class,
+        :opts => opts
+      }
+    end
+
+    # Registers all report printer command line options into the given
+    # OptionParser.
+    # Once the command line is parsed with OptionParser#parse! it will
+    # yield the printer class and the optional command line parameter.
+    #
+    # Note:
+    # If multiple printers are specified in the command line, all are created
+    # and yielded.
+    def self.on_printer_selection(opts)
+      printers.each do |printer|
+        opts.on(*printer[:opts]) do |arg|
+          yield printer[:printer_class], arg
+        end
+      end
+    end
+  end
+end