andyjeffries-rubyrep 1.2.1

data/lib/rubyrep/replication_run.rb

@@ -0,0 +1,142 @@
+require 'timeout'
+
+module RR
+  
+  # Executes a single replication run
+  class ReplicationRun
+
+    # The current Session object
+    attr_accessor :session
+
+    # The current TaskSweeper
+    attr_accessor :sweeper
+
+    # An array of ReplicationDifference which originally failed replication but should be tried one more time
+    def second_chancers
+      @second_chancers ||= []
+    end
+
+    # Returns the current ReplicationHelper; creates it if necessary
+    def helper
+      @helper ||= ReplicationHelper.new(self)
+    end
+
+    # Returns the current replicator; creates it if necessary.
+    def replicator
+      @replicator ||=
+        Replicators.replicators[session.configuration.options[:replicator]].new(helper)
+    end
+
+    # Calls the event filter for the give  difference.
+    # * +diff+: instance of ReplicationDifference
+    # Returns +true+ if replication of the difference should *not* proceed.
+    def event_filtered?(diff)
+      event_filter = helper.options_for_table(diff.changes[:left].table)[:event_filter]
+      if event_filter && event_filter.respond_to?(:before_replicate)
+        not event_filter.before_replicate(
+          diff.changes[:left].table,
+          helper.type_cast(diff.changes[:left].table, diff.changes[:left].key),
+          helper, 
+          diff
+        )
+      else
+        false
+      end
+    end
+
+    # Returns the next available ReplicationDifference.
+    # (Either new unprocessed differences or if not available, the first available 'second chancer'.)
+    #
+    def load_difference
+      @loaders ||= LoggedChangeLoaders.new(session)
+      @loaders.update # ensure the cache of change log records is up-to-date
+      diff = ReplicationDifference.new @loaders
+      diff.load
+      unless diff.loaded? or second_chancers.empty?
+        diff = second_chancers.shift
+      end
+      diff
+    end
+
+    # Executes the replication run.
+    def run
+      return unless [:left, :right].any? do |database|
+        changes_pending = false
+        t = Thread.new do
+          changes_pending = session.send(database).select_one(
+            "select id from #{session.configuration.options[:rep_prefix]}_pending_changes limit 1"
+          ) != nil
+        end
+        t.join session.configuration.options[:database_connection_timeout]
+        changes_pending
+      end
+
+      # Apparently sometimes above check for changes takes already so long, that
+      # the replication run times out.
+      # Check for this and if timed out, return (silently).
+      return if sweeper.terminated?
+
+      success = false
+      begin
+        replicator # ensure that replicator is created and has chance to validate settings
+
+        loop do
+          begin
+            diff = load_difference
+            break unless diff.loaded?
+            break if sweeper.terminated?
+            if diff.type != :no_diff and not event_filtered?(diff)
+              replicator.replicate_difference diff
+            end
+          rescue Exception => e
+            if e.message =~ /violates foreign key constraint|foreign key constraint fails/i and !diff.second_chance?
+              # Note:
+              # Identifying the foreign key constraint violation via regular expression is
+              # database dependent and *dirty*.
+              # It would be better to use the ActiveRecord #translate_exception mechanism.
+              # However as per version 3.0.5 this doesn't work yet properly.
+
+              diff.second_chance = true
+              second_chancers << diff
+            else
+              begin
+                helper.log_replication_outcome diff, e.message,
+                  e.class.to_s + "\n" + e.backtrace.join("\n")
+              rescue Exception => _
+                # if logging to database itself fails, re-raise the original exception
+                raise e
+              end
+            end
+          end
+        end
+        success = true
+      ensure
+        if sweeper.terminated?
+          helper.finalize false
+          session.disconnect_databases
+        else
+          helper.finalize success
+        end
+      end
+    end
+
+    # Installs the current sweeper into the database connections
+    def install_sweeper
+      [:left, :right].each do |database|
+        unless session.send(database).respond_to?(:sweeper)
+          session.send(database).send(:extend, NoisyConnection)
+        end
+        session.send(database).sweeper = sweeper
+      end
+    end
+
+    # Creates a new ReplicationRun instance.
+    # * +session+: the current Session
+    # * +sweeper+: the current TaskSweeper
+    def initialize(session, sweeper)
+      self.session = session
+      self.sweeper = sweeper
+      install_sweeper
+    end
+  end
+end

data/lib/rubyrep/replication_runner.rb

@@ -0,0 +1,166 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'optparse'
+require 'thread'
+require 'monitor'
+
+class Monitor
+  alias lock mon_enter
+  alias unlock mon_exit
+end
+
+module RR
+  # This class implements the functionality of the 'replicate' command.
+  class ReplicationRunner
+
+    CommandRunner.register 'replicate' => {
+      :command => self,
+      :description => 'Starts a replication process'
+    }
+    
+    # Provided options. Possible values:
+    # * +:config_file+: path to config file
+    attr_accessor :options
+
+    # Should be set to +true+ if the replication runner should be terminated.
+    attr_accessor :termination_requested
+
+    # Parses the given command line parameter array.
+    # Returns the status (as per UNIX conventions: 1 if parameters were invalid,
+    # 0 otherwise)
+    def process_options(args)
+      status = 0
+      self.options = {}
+
+      parser = OptionParser.new do |opts|
+        opts.banner = <<EOS
+Usage: #{$0} replicate [options]
+
+  Replicates two databases as per specified configuration file.
+EOS
+        opts.separator ""
+        opts.separator "  Specific options:"
+
+        opts.on("-c", "--config", "=CONFIG_FILE",
+          "Mandatory. Path to configuration file.") do |arg|
+          options[:config_file] = arg
+        end
+
+        opts.on_tail("--help", "Show this message") do
+          $stderr.puts opts
+          self.options = nil
+        end
+      end
+
+      begin
+        parser.parse!(args)
+        if options # this will be +nil+ if the --help option is specified
+          raise("Please specify configuration file") unless options.include?(:config_file)
+        end
+      rescue Exception => e
+        $stderr.puts "Command line parsing failed: #{e}"
+        $stderr.puts parser.help
+        self.options = nil
+        status = 1
+      end
+
+      return status
+    end
+
+    # Returns the active +Session+.
+    # Loads config file and creates session if necessary.
+    def session
+      unless @session
+        unless @config
+          load options[:config_file]
+          @config = Initializer.configuration
+        end
+        @session = Session.new @config
+      end
+      @session
+    end
+
+    # Removes current +Session+.
+    def clear_session
+      @session = nil
+    end
+
+    # Wait for the next replication time
+    def pause_replication
+      @last_run ||= 1.year.ago
+      now = Time.now
+      @next_run = @last_run + session.configuration.options[:replication_interval]
+      unless now >= @next_run
+        waiting_time = @next_run - now
+        @waiter_thread.join waiting_time
+      end
+      @last_run = Time.now
+    end
+
+    # Initializes the waiter thread used for replication pauses and processing
+    # the process TERM signal.
+    def init_waiter
+      @termination_mutex = Monitor.new
+      @termination_mutex.lock
+      @waiter_thread ||= Thread.new {@termination_mutex.lock; self.termination_requested = true}
+      %w(TERM INT).each do |signal|
+        Signal.trap(signal) {puts "\nCaught '#{signal}': Initiating graceful shutdown"; @termination_mutex.unlock}
+      end
+    end
+
+    # Prepares the replication
+    def prepare_replication
+      initializer = ReplicationInitializer.new session
+      initializer.prepare_replication
+    end
+
+    # Executes a single replication run
+    def execute_once
+      session.refresh
+      timeout = session.configuration.options[:database_connection_timeout]
+      terminated = TaskSweeper.timeout(timeout) do |sweeper|
+        run = ReplicationRun.new session, sweeper
+        run.run
+      end.terminated?
+      raise "replication run timed out" if terminated
+    rescue Exception => e
+      clear_session
+      raise e
+    end
+
+    # Executes an endless loop of replication runs
+    def execute
+      init_waiter
+      prepare_replication
+
+      until termination_requested do
+        begin
+          execute_once
+        rescue Exception => e
+          now = Time.now.iso8601
+          $stderr.puts "#{now} Exception caught: #{e}"
+          if @last_exception_message != e.to_s # only print backtrace if something changed
+            @last_exception_message = e.to_s
+            $stderr.puts e.backtrace.map {|line| line.gsub(/^/, "#{' ' * now.length} ")}
+          end
+        end
+        pause_replication
+      end
+    end
+
+    # Entry points for executing a processing run.
+    # args: the array of command line options that were provided by the user.
+    def self.run(args)
+      runner = new
+
+      status = runner.process_options(args)
+      if runner.options
+        runner.execute
+      end
+      status
+    end
+
+  end
+end
+
+

data/lib/rubyrep/replicators/replicators.rb

@@ -0,0 +1,42 @@
+module RR
+  # Replicators are classes that implement the replication policies.
+  # This module provides functionality to register replicators and access the
+  # list of registered replicators.
+  # Each Replicator must register itself with Replicators#register.
+  # Each Replicator must implement the following methods:
+  #
+  #   # Creates a new replicator (A replicator is used for one replication run only)
+  #   #   * sync_helper: a SyncHelper object providing necessary information and functionalities
+  #   def initialize(sync_helper)
+  #
+  #   # Called to sync the provided difference.
+  #   # +difference+ is an instance of +ReplicationDifference+
+  #   def replicate_difference(difference)
+  #
+  #   # Provides default option for the replicator. Optional.
+  #   # Returns a hash with :key => value pairs.
+  #   def self.default_options
+  module Replicators
+    # Returns a Hash of currently registered replicators.
+    # (Empty Hash if no replicators were defined.)
+    def self.replicators
+      @replicators ||= {}
+      @replicators
+    end
+
+    # Returns the correct replicator class as per provided options hash
+    def self.configured_replicator(options)
+      replicators[options[:replicator]]
+    end
+
+    # Registers one or multiple replicators.
+    # syncer_hash is a Hash with
+    #   key::   The adapter symbol as used to reference the replicator
+    #   value:: The class implementing the replicator
+    def self.register(replicator_hash)
+      @replicators ||= {}
+      @replicators.merge! replicator_hash
+    end
+
+  end
+end

data/lib/rubyrep/replicators/two_way_replicator.rb

@@ -0,0 +1,361 @@
+module RR
+  module Replicators
+    # This replicator implements a two way replication.
+    # Options:
+    # * :+left_change_handling+, :+right_change_handling+:
+    #   Handling of records that were changed only in the named database.
+    #   Can be any of the following:
+    #   * :+ignore+: No action.
+    #   * :+replicate+: Updates other database accordingly. *Default* *Setting*
+    #   * +Proc+ object:
+    #     If a Proc object is given, it is responsible for dealing with the
+    #     record. Called with the following parameters:
+    #     * replication_helper: The current ReplicationHelper instance.
+    #     * difference: A ReplicationDifference instance describing the change
+    # * :+replication_conflict_handling+:
+    #   Handling of conflicting record changes. Can be any of the following:
+    #   * :+ignore+: No action. *Default* *Setting*
+    #   * :+left_wins+: The right database is updated accordingly.
+    #   * :+right_wins+: The left database is updated accordingly.
+    #   * :+later_wins+:
+    #     The more recent change is replicated.
+    #     (If both changes have same age: left change is replicated)
+    #   * :+earlier_wins+:
+    #     The less recent change is replicated.
+    #     (If both records have same age: left change is replicated)
+    #   * +Proc+ object:
+    #     If a Proc object is given, it is responsible for dealing with the
+    #     record. Called with the following parameters:
+    #     * replication_helper: The current ReplicationHelper instance.
+    #     * difference: A ReplicationDifference instance describing the changes
+    # * :+logged_replication_events+:
+    #   Specifies which types of replications are logged.
+    #   Is either a single value or an array of multiple ones.
+    #   Default: [:ignored_conflicts]
+    #   Possible values:
+    #   * :+ignored_changes+: log ignored (but not replicated) non-conflict changes
+    #   * :+all_changes+: log all non-conflict changes
+    #   * :+ignored_conflicts+: log ignored (but not replicated) conflicts
+    #   * :+all_conflicts+: log all conflicts
+    #
+    # Example of using a Proc object for custom behaviour:
+    #   lambda do |rep_helper, diff|
+    #     # if specified as replication_conflict_handling option, logs all
+    #     # conflicts to a text file
+    #     File.open('/var/log/rubyrep_conflict_log', 'a') do |f|
+    #       f.puts <<-end_str
+    #         #{Time.now}: conflict
+    #         * in table #{diff.changes[:left].table}
+    #         * for record '#{diff.changes[:left].key}'
+    #         * change type in left db: '#{diff.changes[:left].type}'
+    #         * change type in right db: '#{diff.changes[:right].type}'
+    #       end_str
+    #     end
+    #   end
+    class TwoWayReplicator
+      
+      # Register the syncer
+      Replicators.register :two_way => self
+
+      # The current ReplicationHelper object
+      attr_accessor :rep_helper
+
+      # Provides default option for the replicator. Optional.
+      # Returns a hash with key => value pairs.
+      def self.default_options
+        {
+          :left_change_handling => :replicate,
+          :right_change_handling => :replicate,
+          :replication_conflict_handling => :ignore,
+          :logged_replication_events => [:ignored_conflicts],
+        }
+      end
+
+      # Checks if an option is configured correctly. Raises an ArgumentError if not.
+      # * +table_spec+: the table specification to which the option belongs. May be +nil+.
+      # * +valid_option_values+: array of valid option values
+      # * +option_key+: the key of the option that is to be checked
+      # * +option_value+: the value of the option that is to be checked
+      def verify_option(table_spec, valid_option_values, option_key, option_value)
+        unless valid_option_values.include? option_value
+          message = ""
+          message << "#{table_spec.inspect}: " if table_spec
+          message << "#{option_value.inspect} not a valid #{option_key.inspect} option"
+          raise ArgumentError.new(message)
+        end
+      end
+
+      # Verifies if the :+left_change_handling+ / :+right_change_handling+
+      # options are valid.
+      # Raises an ArgumentError if an option is invalid
+      def validate_change_handling_options
+        [:left_change_handling, :right_change_handling].each do |key|
+          rep_helper.session.configuration.each_matching_option(key) do |table_spec, value|
+            unless value.respond_to? :call
+              verify_option table_spec, [:ignore, :replicate], key, value
+            end
+          end
+        end
+      end
+
+      # Verifies if the given :+replication_conflict_handling+ options are valid.
+      # Raises an ArgumentError if an option is invalid.
+      def validate_conflict_handling_options
+        rep_helper.session.configuration.each_matching_option(:replication_conflict_handling) do |table_spec, value|
+          unless value.respond_to? :call
+            verify_option table_spec,
+              [:ignore, :left_wins, :right_wins, :later_wins, :earlier_wins],
+              :replication_conflict_handling, value
+          end
+        end
+      end
+
+      # Verifies if the given :+replication_logging+ option /options is / are valid.
+      # Raises an ArgumentError if invalid
+      def validate_logging_options
+        rep_helper.session.configuration.each_matching_option(:logged_replication_events) do |table_spec, values|
+          values = [values].flatten # ensure that I have an array
+          values.each do |value|
+            verify_option table_spec,
+              [:ignored_changes, :all_changes, :ignored_conflicts, :all_conflicts],
+              :logged_replication_events, value
+          end
+        end
+      end
+
+      # Initializes the TwoWayReplicator
+      # Raises an ArgumentError if any of the replication options is invalid.
+      #
+      # Parameters:
+      # * rep_helper:
+      #   The ReplicationHelper object providing information and utility functions.
+      def initialize(rep_helper)
+        self.rep_helper = rep_helper
+
+        validate_change_handling_options
+        validate_conflict_handling_options
+        validate_logging_options
+      end
+
+      # Shortcut to calculate the "other" database.
+      OTHER_SIDE = {
+        :left => :right,
+        :right => :left
+      }
+
+      # Specifies how to clear conflicts.
+      # The outer hash keys describe the type of the winning change.
+      # The inner hash keys describe the type of the loosing change.
+      # The inser hash values describe the action to take on the loosing side.
+      CONFLICT_STATE_MATRIX = {
+        :insert => {:insert => :update, :update => :update, :delete => :insert},
+        :update => {:insert => :update, :update => :update, :delete => :insert},
+        :delete => {:insert => :delete, :update => :delete, :delete => :delete}
+      }
+
+      # Helper function that clears a conflict by taking the change from the
+      # specified winning database and updating the other database accordingly.
+      # * +source_db+: the winning database (either :+left+ or :+right+)
+      # * +diff+: the ReplicationDifference instance
+      # * +remaining_attempts+: the number of remaining replication attempts for this difference
+      def clear_conflict(source_db, diff, remaining_attempts)
+        source_change = diff.changes[source_db]
+        target_db = OTHER_SIDE[source_db]
+        target_change = diff.changes[target_db]
+
+        target_action = CONFLICT_STATE_MATRIX[source_change.type][target_change.type]
+        source_key = source_change.type == :update ? source_change.new_key : source_change.key
+        target_key = target_change.type == :update ? target_change.new_key : target_change.key
+        case target_action
+        when :insert
+          attempt_insert source_db, diff, remaining_attempts, source_key
+        when :update
+          attempt_update source_db, diff, remaining_attempts, source_key, target_key
+        when :delete
+          attempt_delete source_db, diff, remaining_attempts, target_key
+        end
+      end
+
+      # Logs replication of the specified difference as per configured
+      # :+replication_conflict_logging+ / :+left_change_logging+ / :+right_change_logging+ options.
+      # * +winner+: Either the winner database (:+left+ or :+right+) or :+ignore+
+      # * +diff+: the ReplicationDifference instance
+      def log_replication_outcome(winner, diff)
+        options = rep_helper.options_for_table(diff.changes[:left].table)
+        option_values = [options[:logged_replication_events]].flatten # make sure I have an array
+        if diff.type == :conflict
+          return unless option_values.include?(:all_conflicts) or option_values.include?(:ignored_conflicts)
+          return if winner != :ignore and not option_values.include?(:all_conflicts)
+          outcome = {:left => 'left_won', :right => 'right_won', :ignore => 'ignored'}[winner]
+        else
+          return unless option_values.include?(:all_changes) or option_values.include?(:ignored_changes)
+          return if winner != :ignore and not option_values.include?(:all_changes)
+          outcome = winner == :ignore ? 'ignored' : 'replicated'
+        end
+        rep_helper.log_replication_outcome diff, outcome
+      end
+
+      # How often a replication will be attempted (in case it fails because the
+      # record in question was removed from the source or inserted into the
+      # target database _after_ the ReplicationDifference was loaded
+      MAX_REPLICATION_ATTEMPTS = 2
+
+      # Attempts to read the specified record from the source database and insert
+      # it into the target database.
+      # Retries if insert fails due to missing source or suddenly existing target
+      # record.
+      # * +source_db+: either :+left+ or :+right+ - source database of replication
+      # * +diff+: the current ReplicationDifference instance
+      # * +remaining_attempts+: the number of remaining replication attempts for this difference
+      # * +source_key+: a column_name => value hash identifying the source record
+      def attempt_insert(source_db, diff, remaining_attempts, source_key)
+        source_change = diff.changes[source_db]
+        source_table = source_change.table
+        target_db = OTHER_SIDE[source_db]
+        target_table = rep_helper.corresponding_table(source_db, source_table)
+
+        values = rep_helper.load_record source_db, source_table, source_key
+        if values == nil
+          diff.amend
+          replicate_difference diff, remaining_attempts - 1, "source record for insert vanished"
+        else
+          attempt_change('insert', source_db, target_db, diff, remaining_attempts) do
+            rep_helper.insert_record target_db, target_table, values
+            log_replication_outcome source_db, diff
+          end
+        end
+      end
+
+      # Attempts to read the specified record from the source database and update
+      # the specified record in the target database.
+      # Retries if update fails due to missing source
+      # * +source_db+: either :+left+ or :+right+ - source database of replication
+      # * +diff+: the current ReplicationDifference instance
+      # * +remaining_attempts+: the number of remaining replication attempts for this difference
+      # * +source_key+: a column_name => value hash identifying the source record
+      # * +target_key+: a column_name => value hash identifying the source record
+      def attempt_update(source_db, diff, remaining_attempts, source_key, target_key)
+        source_change = diff.changes[source_db]
+        source_table = source_change.table
+        target_db = OTHER_SIDE[source_db]
+        target_table = rep_helper.corresponding_table(source_db, source_table)
+
+        values = rep_helper.load_record source_db, source_table, source_key
+        if values == nil
+          diff.amend
+          replicate_difference diff, remaining_attempts - 1, "source record for update vanished"
+        else
+          attempt_change('update', source_db, target_db, diff, remaining_attempts) do
+            number_updated = rep_helper.update_record target_db, target_table, values, target_key
+            if number_updated == 0
+              diff.amend
+              replicate_difference diff, remaining_attempts - 1, "target record for update vanished"
+            else
+              log_replication_outcome source_db, diff
+            end
+          end
+        end
+      end
+
+      # Helper for execution of insert / update / delete attempts.
+      # Wraps those attempts into savepoints and handles exceptions.
+      #
+      # Note:
+      # Savepoints have to be used for PostgreSQL (as a failed SQL statement
+      # will otherwise invalidate the complete transaction.)
+      #
+      # * +action+: short description of change (e. g.: "update" or "delete")
+      # * +source_db+: either :+left+ or :+right+ - source database of replication
+      # * +target_db+: either :+left+ or :+right+ - target database of replication
+      # * +diff+: the current ReplicationDifference instance
+      # * +remaining_attempts+: the number of remaining replication attempts for this difference
+      def attempt_change(action, source_db, target_db, diff, remaining_attempts)
+        begin
+          rep_helper.session.send(target_db).execute "savepoint rr_#{action}_#{remaining_attempts}"
+          yield
+          unless rep_helper.new_transaction?
+            rep_helper.session.send(target_db).execute "release savepoint rr_#{action}_#{remaining_attempts}"
+          end
+        rescue Exception => e
+          rep_helper.session.send(target_db).execute "rollback to savepoint rr_#{action}_#{remaining_attempts}"
+          diff.amend
+          replicate_difference diff, remaining_attempts - 1,
+            "#{action} failed with #{e.message}"
+        end
+      end
+
+      # Attempts to delete the source record from the target database.
+      # E. g. if +source_db is :+left+, then the record is deleted in database
+      # :+right+.
+      # * +source_db+: either :+left+ or :+right+ - source database of replication
+      # * +diff+: the current ReplicationDifference instance
+      # * +remaining_attempts+: the number of remaining replication attempts for this difference
+      # * +target_key+: a column_name => value hash identifying the source record
+      def attempt_delete(source_db, diff, remaining_attempts, target_key)
+        change = diff.changes[source_db]
+        target_db = OTHER_SIDE[source_db]
+        target_table = rep_helper.corresponding_table(source_db, change.table)
+
+        attempt_change('delete', source_db, target_db, diff, remaining_attempts) do
+          number_updated = rep_helper.delete_record target_db, target_table, target_key
+          if number_updated == 0
+            diff.amend
+            replicate_difference diff, remaining_attempts - 1, "target record for delete vanished"
+          else
+            log_replication_outcome source_db, diff
+          end
+        end
+      end
+
+      # Called to replicate the specified difference.
+      # * :+diff+: ReplicationDifference instance
+      # * :+remaining_attempts+: how many more times a replication will be attempted
+      # * :+previous_failure_description+: why the previous replication attempt failed
+      def replicate_difference(diff, remaining_attempts = MAX_REPLICATION_ATTEMPTS, previous_failure_description = nil)
+        raise Exception, previous_failure_description || "max replication attempts exceeded" if remaining_attempts == 0
+        options = rep_helper.options_for_table(diff.changes[:left].table)
+        if diff.type == :left or diff.type == :right
+          key = diff.type == :left ? :left_change_handling : :right_change_handling
+          option = options[key]
+
+          if option == :ignore
+            log_replication_outcome :ignore, diff
+          elsif option == :replicate
+            source_db = diff.type
+
+            change = diff.changes[source_db]
+
+            case change.type
+            when :insert
+              attempt_insert source_db, diff, remaining_attempts, change.key
+            when :update
+              attempt_update source_db, diff, remaining_attempts, change.new_key, change.key
+            when :delete
+              attempt_delete source_db, diff, remaining_attempts, change.key
+            end
+          else # option must be a Proc
+            option.call rep_helper, diff
+          end
+        elsif diff.type == :conflict
+          option = options[:replication_conflict_handling]
+          if option == :ignore
+            log_replication_outcome :ignore, diff
+          elsif option == :left_wins
+            clear_conflict :left, diff, remaining_attempts
+          elsif option == :right_wins
+            clear_conflict :right, diff, remaining_attempts
+          elsif option == :later_wins
+            winner_db = diff.changes[:left].last_changed_at >= diff.changes[:right].last_changed_at ? :left : :right
+            clear_conflict winner_db, diff, remaining_attempts
+          elsif option == :earlier_wins
+            winner_db = diff.changes[:left].last_changed_at <= diff.changes[:right].last_changed_at ? :left : :right
+            clear_conflict winner_db, diff, remaining_attempts
+          else # option must be a Proc
+            option.call rep_helper, diff
+          end
+        end
+      end
+      
+    end
+  end
+end