andyjeffries-rubyrep 1.2.1

data/lib/rubyrep/sync_helper.rb

@@ -0,0 +1,121 @@
+module RR
+
+  # Provides helper functionality for the table syncers.
+  # The methods exposed by this class are intended to provide a stable interface
+  # for third party syncers.
+  class SyncHelper
+
+    include LogHelper
+
+    # The current +TableSync+ instance
+    attr_accessor :table_sync
+
+    # The active +Session+
+    def session; table_sync.session; end
+
+    # Name of the left table
+    def left_table; table_sync.left_table; end
+
+    # Name of the right table
+    def right_table; table_sync.right_table; end
+
+    # A hash with
+    # :+left+: name of the table in the left database
+    # :+right+: name of the table in the right database
+    def tables
+      @tables ||= {:left => left_table, :right => right_table}
+    end
+
+    # Given a column_name => value hash of a full row, returns a
+    # column_name => value hash of the primary key columns.
+    # * +row+: the full row
+    # Returns
+    def extract_key(row)
+      row.reject {|column, value| not primary_key_names.include? column }
+    end
+
+    # Sync options for the current table sync
+    def sync_options; @sync_options ||= table_sync.sync_options; end
+
+    # Delegates to Committers::BufferedCommitter#insert_record
+    def insert_record(database, table, values)
+      committer.insert_record(database, tables[database], values)
+    end
+
+    # Delegates to Committers::BufferedCommitter#update_record
+    def update_record(database, table, values, old_key = nil)
+      committer.update_record(database, tables[database], values, old_key)
+    end
+
+    # Delegates to Committers::BufferedCommitter#delete_record
+    def delete_record(database, table, values)
+      committer.delete_record(database, tables[database], values)
+    end
+
+    # Return the committer, creating it if not yet there.
+    def committer
+      unless @committer
+        committer_class = Committers::committers[sync_options[:committer]]
+        @committer = committer_class.new(session)
+      end
+      @committer
+    end
+    private :committer
+
+    # Checks if the event log table already exists and creates it if necessary
+    def ensure_event_log
+      unless @ensured_event_log
+        ReplicationInitializer.new(session).ensure_event_log
+        @ensured_event_log = true
+      end
+    end
+
+    # Returns an array of primary key names for the synced table
+    def primary_key_names
+      @primary_key_names ||= session.left.primary_key_names(left_table)
+    end
+    private :primary_key_names
+
+    # Logs the outcome of a replication into the replication log table.
+    # * +row+: a column_name => value hash for at least the primary keys of the record
+    # * +type+: string describing the type of the sync
+    # * +outcome+: string describing what's done about the sync
+    # * +details+: string with further details regarding the sync
+    def log_sync_outcome(row, type, outcome, details = nil)
+      ensure_event_log
+      if primary_key_names.size == 1
+        key = row[primary_key_names[0]]
+      else
+        key_parts = primary_key_names.map do |column_name|
+          %Q("#{column_name}"=>#{row[column_name].to_s.inspect})
+        end
+        key = key_parts.join(', ')
+      end
+      sync_outcome, sync_details = fit_description_columns(outcome, details)
+
+      session.left.insert_record "#{sync_options[:rep_prefix]}_logged_events", {
+        :activity => 'sync',
+        :change_table => left_table,
+        :diff_type => type.to_s,
+        :change_key => key,
+        :left_change_type => nil,
+        :right_change_type => nil,
+        :description => sync_outcome,
+        :long_description => sync_details,
+        :event_time => Time.now,
+        :diff_dump => nil
+      }
+    end
+
+    # Asks the committer (if it exists) to finalize any open transactions
+    # +success+ should be true if there were no problems, false otherwise.
+    def finalize(success = true)
+      @committer.finalize(success) if @committer
+    end
+    
+    # Creates a new SyncHelper for the given +TableSync+ instance.
+    def initialize(table_sync)
+      self.table_sync = table_sync
+    end
+  end
+end

data/lib/rubyrep/sync_runner.rb

@@ -0,0 +1,31 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+
+module RR
+  # This class implements the functionality of the rrsync.rb command.
+  class SyncRunner < BaseRunner
+
+    CommandRunner.register 'sync' => {
+      :command => self,
+      :description => 'Syncs records between databases'
+    }
+
+    # Returns summary description string for the scan command.
+    def summary_description
+      "Syncs the differences of the specified tables between both databases."
+    end
+
+    # Creates the correct scan class.
+    # Parameters as defined under BaseRunner#create_processor
+    def create_processor(left_table, right_table)
+      TableSync.new session, left_table, right_table
+    end
+
+    # Reorders the table pairs to avoid foreign key conflicts.
+    # More information on this methods at BaseRunner#prepare_table_pairs.
+    def prepare_table_pairs(table_pairs)
+      session.sort_table_pairs(table_pairs)
+    end
+  end
+end
+
+

data/lib/rubyrep/syncers/syncers.rb

@@ -0,0 +1,112 @@
+module RR
+  # Syncers are classes that implement the sync policies.
+  # This module provides functionality to register syncers and access the
+  # list of registered syncers.
+  # Each Syncer must register itself with Syncers#register.
+  # Each Syncer must implement the following methods:
+  #
+  #   # Creates a new syncer (A syncer is used for one table sync only)
+  #   #   * sync_helper: a SyncHelper object providing necessary information and functionalities
+  #   def initialize(sync_helper)
+  #
+  #   # Called to sync the provided difference.
+  #   # See DirectTableScan#run for a description of the +type+ and +row+ parameters.
+  #   def sync_difference(type, row)
+  #
+  #   # Provides default option for the syncer. Optional.
+  #   # Returns a hash with :key => value pairs.
+  #   def self.default_options
+  module Syncers
+    # Returns a Hash of currently registered syncers.
+    # (Empty Hash if no syncers were defined.)
+    def self.syncers
+      @syncers ||= {}
+      @syncers
+    end
+  
+    # Registers one or multiple syncers.
+    # syncer_hash is a Hash with
+    #   key::   The adapter symbol as used to reference the syncer
+    #   value:: The class implementing the syncer
+    def self.register(syncer_hash)
+      @syncers ||= {}
+      @syncers.merge! syncer_hash
+    end
+
+    # Returns the correct syncer class as per provided options hash
+    def self.configured_syncer(options)
+      syncer_id = options[:syncer]
+      syncer_id ||= options[:replicator]
+      syncers[syncer_id]
+    end
+    
+    # This syncer implements a one way sync.
+    # Syncer options relevant for this syncer:
+    #   * +:direction+: Sync direction. Possible values:
+    #     * +:left+
+    #     * +:right+
+    #   * +:delete+: Default: false. If true, deletes in the target database all
+    #                records _not_ existing in the source database.
+    #   * +:update+: If true (default), update records in the target database
+    #                if different.
+    #   * +:insert+: If true (default), copy over records not existing in the
+    #                target database.
+    class OneWaySyncer
+      
+      # Register the syncer
+      Syncers.register :one_way => self
+
+      # The current SyncHelper object
+      attr_accessor :sync_helper
+
+      # ID of source database (either :left or :right)
+      attr_accessor :source
+
+      # ID of target database (either :left or :right)
+      attr_accessor :target
+
+      # Array index to source row in case #sync_difference +type+ is :conflict.
+      # (As in that case the +row+ parameter is an array of left and right records.)
+      attr_accessor :source_record_index
+      
+      # Provides default option for the syncer. Optional.
+      # Returns a hash with :key => value pairs.
+      def self.default_options
+        {
+          :direction => :right,
+          :delete => false, :update => true, :insert => true
+        }
+      end
+
+      # Initializes the syncer
+      #   * sync_helper: The SyncHelper object provided information and utility
+      #                  functions.
+      def initialize(sync_helper)
+        self.sync_helper = sync_helper
+        self.source = sync_helper.sync_options[:direction] == :left ? :right : :left
+        self.target = sync_helper.sync_options[:direction] == :left ? :left : :right
+        self.source_record_index = sync_helper.sync_options[:direction] == :left ? 1 : 0
+      end
+
+      # Called to sync the provided difference.
+      # See DirectTableScan#run for a description of the +type+ and +row+ parameters.
+      def sync_difference(type, row)
+        case type
+        when source
+          if sync_helper.sync_options[:insert]
+            sync_helper.insert_record target, sync_helper.tables[target], row
+          end
+        when target
+          if sync_helper.sync_options[:delete]
+            sync_helper.delete_record target, sync_helper.tables[target], row
+          end
+        when :conflict
+          if sync_helper.sync_options[:update]
+            sync_helper.update_record target, sync_helper.tables[target], row[source_record_index]
+          end
+        end
+      end
+    end
+    
+  end
+end

data/lib/rubyrep/syncers/two_way_syncer.rb

@@ -0,0 +1,174 @@
+module RR
+  module Syncers
+    # This syncer implements a two way sync.
+    # Syncer options relevant for this syncer:
+    # * :+left_record_handling+, :+right_record_handling+:
+    #   Handling of records only existing only in the named database.
+    #   Can be any of the following:
+    #   * :+ignore+: No action.
+    #   * :+delete+: Delete from the source database.
+    #   * :+insert+: Insert in the target database. *Default* *Setting*
+    #   * +Proc+ object:
+    #     If a Proc object is given, it is responsible for dealing with the
+    #     record. Called with the following parameters:
+    #     * sync_helper: The current SyncHelper instance.
+    #     * type: :+left+ or :+right+ to designate source database
+    #     * row: column_name => value hash representing the row
+    # * :+sync_conflict_handling+:
+    #   Handling of conflicting records. Can be any of the following:
+    #   * :+ignore+: No action. *Default* *Setting*
+    #   * :+left_wins+: Update right database with the field values in the left db.
+    #   * :+right_wins+: Update left database with the field values in the right db.
+    #   * +Proc+ object:
+    #     If a Proc object is given, it is responsible for dealing with the
+    #     record. Called with the following parameters:
+    #     * sync_helper: The current SyncHelper instance.
+    #     * type: always :+conflict+
+    #     * rows: A two element array of rows (column_name => value hashes).
+    #       First left, than right record.
+    # * :+logged_sync_events+:
+    #   Specifies which types of syncs are logged.
+    #   Is either a single value or an array of multiple ones.
+    #   Default: [:ignored_conflicts]
+    #   Possible values:
+    #   * :+ignored_changes+: log ignored (but not synced) non-conflict changes
+    #   * :+all_changes+: log all non-conflict changes
+    #   * :+ignored_conflicts+: log ignored (but not synced) conflicts
+    #   * :+all_conflicts+: log all conflicts
+    #
+    # Example of using a Proc object:
+    #   lambda do |sync_helper, type, row|
+    #     # delete records existing only in the left database.
+    #     sync_helper.delete(type, row) if type == :left
+    #   end
+    class TwoWaySyncer
+      
+      # Register the syncer
+      Syncers.register :two_way => self
+
+      # The current SyncHelper object
+      attr_accessor :sync_helper
+
+      # Provides default option for the syncer. Optional.
+      # Returns a hash with key => value pairs.
+      def self.default_options
+        {
+          :left_record_handling => :insert,
+          :right_record_handling => :insert,
+          :sync_conflict_handling => :ignore,
+          :logged_sync_events => [:ignored_conflicts]
+        }
+      end
+
+      # Verifies if the given :+left_record_handling+ / :+right_record_handling+
+      # option is valid.
+      # Raises an ArgumentError if option is invalid
+      def validate_left_right_record_handling_option(option)
+        unless option.respond_to? :call
+          unless [:ignore, :delete, :insert].include? option
+            raise ArgumentError.new("#{option.inspect} not a valid :left_record_handling / :right_record_handling option")
+          end
+        end
+      end
+
+      # Verifies if the given :+sync_conflict_handling+ option is valid.
+      # Raises an ArgumentError if option is invalid
+      def validate_conflict_handling_option(option)
+        unless option.respond_to? :call
+          unless [:ignore, :right_wins, :left_wins].include? option
+            raise ArgumentError.new("#{option.inspect} not a valid :sync_conflict_handling option")
+          end
+        end
+      end
+
+      # Verifies if the given :+replication_logging+ option /options is / are valid.
+      # Raises an ArgumentError if invalid
+      def validate_logging_options(options)
+        values = [options].flatten # ensure that I have an array
+        values.each do |value|
+          unless [:ignored_changes, :all_changes, :ignored_conflicts, :all_conflicts].include? value
+            raise ArgumentError.new("#{value.inspect} not a valid :logged_sync_events option")
+          end
+        end
+      end
+
+      # Initializes the syncer
+      # * sync_helper:
+      #   The SyncHelper object provided information and utility functions.
+      # Raises an ArgumentError if any of the option in sync_helper.sync_options
+      # is invalid.
+      def initialize(sync_helper)
+        validate_left_right_record_handling_option sync_helper.sync_options[:left_record_handling]
+        validate_left_right_record_handling_option sync_helper.sync_options[:right_record_handling]
+        validate_conflict_handling_option sync_helper.sync_options[:sync_conflict_handling]
+        validate_logging_options sync_helper.sync_options[:logged_sync_events]
+        
+        self.sync_helper = sync_helper
+      end
+
+      # Sync type descriptions that are written into the event log
+      TYPE_DESCRIPTIONS = {
+        :left => 'left_record',
+        :right => 'right_record',
+        :conflict => 'conflict'
+      }
+
+      # Returns the :logged_sync_events option values.
+      def log_option_values
+        @log_option_values ||= [sync_helper.sync_options[:logged_sync_events]].flatten
+      end
+      private :log_option_values
+
+      # Logs a sync event into the event log table as per configuration options.
+      # * +type+: Refer to DirectTableScan#run for a description
+      # * +action+: the sync action that is executed
+      #   (The :+left_record_handling+ / :+right_record_handling+ or
+      #   :+sync_conflict_handling+ option)
+      # * +row+: Refer to DirectTableScan#run for a description
+      def log_sync_outcome(type, action, row)
+        if type == :conflict
+          return unless log_option_values.include?(:all_conflicts) or log_option_values.include?(:ignored_conflicts)
+          return if action != :ignore and not log_option_values.include?(:all_conflicts)
+          row = row[0] # Extract left row from row array
+        else
+          return unless log_option_values.include?(:all_changes) or log_option_values.include?(:ignored_changes)
+          return if action != :ignore and not log_option_values.include?(:all_changes)
+        end
+
+        sync_helper.log_sync_outcome row, TYPE_DESCRIPTIONS[type], action
+      end
+
+      # Called to sync the provided difference.
+      # See DirectTableScan#run for a description of the +type+ and +row+ parameters.
+      def sync_difference(type, row)
+        if type == :left or type == :right
+          option_key = type == :left ? :left_record_handling : :right_record_handling
+          option = sync_helper.sync_options[option_key]
+          log_sync_outcome type, option, row unless option.respond_to?(:call)
+          if option == :ignore
+            # nothing to do
+          elsif option == :delete
+            sync_helper.delete_record type, sync_helper.tables[type], row
+          elsif option == :insert
+            target = (type == :left ? :right : :left)
+            sync_helper.insert_record target, sync_helper.tables[target], row
+          else #option must be a Proc
+            option.call sync_helper, type, row
+          end
+        else
+          option = sync_helper.sync_options[:sync_conflict_handling]
+          log_sync_outcome type, option, row unless option.respond_to?(:call)
+          if option == :ignore
+            # nothing to do
+          elsif option == :right_wins
+            sync_helper.update_record :left, sync_helper.tables[:left], row[1]
+          elsif option == :left_wins
+            sync_helper.update_record :right, sync_helper.tables[:right], row[0]
+          else #option must be a Proc
+            option.call sync_helper, type, row
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rubyrep/table_scan.rb

@@ -0,0 +1,54 @@
+module RR
+
+  # Shared functionality for DirectTableScan and ProxiedTableScan
+  class TableScan
+    include TableScanHelper
+
+    # The current Session object
+    attr_accessor :session 
+      
+    # Name of the left table
+    attr_accessor :left_table
+    
+    # Name of the right table
+    attr_accessor :right_table
+
+    # Cached array of primary key names
+    attr_accessor :primary_key_names
+
+    # Receives the active ScanProgressPrinters class
+    attr_accessor :progress_printer
+
+    # Returns a hash of scan options for this table scan.
+    def scan_options
+      @scan_options ||= session.configuration.options_for_table(left_table)
+    end
+
+    # Inform new progress to progress printer
+    # +steps+ is the number of processed records.
+    def update_progress(steps)
+      return unless progress_printer
+      unless @progress_printer_instance
+        total_records =
+          session.left.select_one("select count(*) as n from #{session.left.quote_table_name(left_table)}")['n'].to_i +
+          session.right.select_one("select count(*) as n from #{session.right.quote_table_name(right_table)}")['n'].to_i
+        @progress_printer_instance = progress_printer.new(total_records, session, left_table, right_table)
+      end
+      @progress_printer_instance.step(steps)
+    end
+    
+    # Creates a new DirectTableScan instance
+    #   * session: a Session object representing the current database session
+    #   * left_table: name of the table in the left database
+    #   * right_table: name of the table in the right database. If not given, same like left_table
+    def initialize(session, left_table, right_table = nil)
+      if session.left.primary_key_names(left_table).empty?
+        raise "Table '#{left_table}' doesn't have a primary key. Cannot scan."
+      end
+      
+      self.session, self.left_table, self.right_table = session, left_table, right_table
+      self.right_table ||= self.left_table
+      self.primary_key_names = session.left.primary_key_names left_table
+    end
+  end
+end