rubyrep 1.0.0

data/lib/rubyrep/syncers/two_way_syncer.rb

@@ -0,0 +1,174 @@
+module RR
+  module Syncers
+    # This syncer implements a two way sync.
+    # Syncer options relevant for this syncer:
+    # * :+left_record_handling+, :+right_record_handling+:
+    #   Handling of records only existing only in the named database.
+    #   Can be any of the following:
+    #   * :+ignore+: No action.
+    #   * :+delete+: Delete from the source database.
+    #   * :+insert+: Insert in the target database. *Default* *Setting*
+    #   * +Proc+ object:
+    #     If a Proc object is given, it is responsible for dealing with the
+    #     record. Called with the following parameters:
+    #     * sync_helper: The current SyncHelper instance.
+    #     * type: :+left+ or :+right+ to designate source database
+    #     * row: column_name => value hash representing the row
+    # * :+sync_conflict_handling+:
+    #   Handling of conflicting records. Can be any of the following:
+    #   * :+ignore+: No action. *Default* *Setting*
+    #   * :+left_wins+: Update right database with the field values in the left db.
+    #   * :+right_wins+: Update left database with the field values in the right db.
+    #   * +Proc+ object:
+    #     If a Proc object is given, it is responsible for dealing with the
+    #     record. Called with the following parameters:
+    #     * sync_helper: The current SyncHelper instance.
+    #     * type: always :+conflict+
+    #     * rows: A two element array of rows (column_name => value hashes).
+    #       First left, than right record.
+    # * :+logged_sync_events+:
+    #   Specifies which types of syncs are logged.
+    #   Is either a single value or an array of multiple ones.
+    #   Default: [:ignored_conflicts]
+    #   Possible values:
+    #   * :+ignored_changes+: log ignored (but not synced) non-conflict changes
+    #   * :+all_changes+: log all non-conflict changes
+    #   * :+ignored_conflicts+: log ignored (but not synced) conflicts
+    #   * :+all_conflicts+: log all conflicts
+    #
+    # Example of using a Proc object:
+    #   lambda do |sync_helper, type, row|
+    #     # delete records existing only in the left database.
+    #     sync_helper.delete(type, row) if type == :left
+    #   end
+    class TwoWaySyncer
+      
+      # Register the syncer
+      Syncers.register :two_way => self
+
+      # The current SyncHelper object
+      attr_accessor :sync_helper
+
+      # Provides default option for the syncer. Optional.
+      # Returns a hash with key => value pairs.
+      def self.default_options
+        {
+          :left_record_handling => :insert,
+          :right_record_handling => :insert,
+          :sync_conflict_handling => :ignore,
+          :logged_sync_events => [:ignored_conflicts]
+        }
+      end
+
+      # Verifies if the given :+left_record_handling+ / :+right_record_handling+
+      # option is valid.
+      # Raises an ArgumentError if option is invalid
+      def validate_left_right_record_handling_option(option)
+        unless option.respond_to? :call
+          unless [:ignore, :delete, :insert].include? option
+            raise ArgumentError.new("#{option.inspect} not a valid :left_record_handling / :right_record_handling option")
+          end
+        end
+      end
+
+      # Verifies if the given :+sync_conflict_handling+ option is valid.
+      # Raises an ArgumentError if option is invalid
+      def validate_conflict_handling_option(option)
+        unless option.respond_to? :call
+          unless [:ignore, :right_wins, :left_wins].include? option
+            raise ArgumentError.new("#{option.inspect} not a valid :sync_conflict_handling option")
+          end
+        end
+      end
+
+      # Verifies if the given :+replication_logging+ option /options is / are valid.
+      # Raises an ArgumentError if invalid
+      def validate_logging_options(options)
+        values = [options].flatten # ensure that I have an array
+        values.each do |value|
+          unless [:ignored_changes, :all_changes, :ignored_conflicts, :all_conflicts].include? value
+            raise ArgumentError.new("#{value.inspect} not a valid :logged_sync_events option")
+          end
+        end
+      end
+
+      # Initializes the syncer
+      # * sync_helper:
+      #   The SyncHelper object provided information and utility functions.
+      # Raises an ArgumentError if any of the option in sync_helper.sync_options
+      # is invalid.
+      def initialize(sync_helper)
+        validate_left_right_record_handling_option sync_helper.sync_options[:left_record_handling]
+        validate_left_right_record_handling_option sync_helper.sync_options[:right_record_handling]
+        validate_conflict_handling_option sync_helper.sync_options[:sync_conflict_handling]
+        validate_logging_options sync_helper.sync_options[:logged_sync_events]
+        
+        self.sync_helper = sync_helper
+      end
+
+      # Sync type descriptions that are written into the event log
+      TYPE_DESCRIPTIONS = {
+        :left => 'left_record',
+        :right => 'right_record',
+        :conflict => 'conflict'
+      }
+
+      # Returns the :logged_sync_events option values.
+      def log_option_values
+        @log_option_values ||= [sync_helper.sync_options[:logged_sync_events]].flatten
+      end
+      private :log_option_values
+
+      # Logs a sync event into the event log table as per configuration options.
+      # * +type+: Refer to DirectTableScan#run for a description
+      # * +action+: the sync action that is executed
+      #   (The :+left_record_handling+ / :+right_record_handling+ or
+      #   :+sync_conflict_handling+ option)
+      # * +row+: Refer to DirectTableScan#run for a description
+      def log_sync_outcome(type, action, row)
+        if type == :conflict
+          return unless log_option_values.include?(:all_conflicts) or log_option_values.include?(:ignored_conflicts)
+          return if action != :ignore and not log_option_values.include?(:all_conflicts)
+          row = row[0] # Extract left row from row array
+        else
+          return unless log_option_values.include?(:all_changes) or log_option_values.include?(:ignored_changes)
+          return if action != :ignore and not log_option_values.include?(:all_changes)
+        end
+
+        sync_helper.log_sync_outcome row, TYPE_DESCRIPTIONS[type], action
+      end
+
+      # Called to sync the provided difference.
+      # See DirectTableScan#run for a description of the +type+ and +row+ parameters.
+      def sync_difference(type, row)
+        if type == :left or type == :right
+          option_key = type == :left ? :left_record_handling : :right_record_handling
+          option = sync_helper.sync_options[option_key]
+          log_sync_outcome type, option, row unless option.respond_to?(:call)
+          if option == :ignore
+            # nothing to do
+          elsif option == :delete
+            sync_helper.delete_record type, row
+          elsif option == :insert
+            target = (type == :left ? :right : :left)
+            sync_helper.insert_record target, row
+          else #option must be a Proc
+            option.call sync_helper, type, row
+          end
+        else
+          option = sync_helper.sync_options[:sync_conflict_handling]
+          log_sync_outcome type, option, row unless option.respond_to?(:call)
+          if option == :ignore
+            # nothing to do
+          elsif option == :right_wins
+            sync_helper.update_record :left, row[1]
+          elsif option == :left_wins
+            sync_helper.update_record :right, row[0]
+          else #option must be a Proc
+            option.call sync_helper, type, row
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rubyrep/table_scan.rb

@@ -0,0 +1,54 @@
+module RR
+
+  # Shared functionality for DirectTableScan and ProxiedTableScan
+  class TableScan
+    include TableScanHelper
+
+    # The current Session object
+    attr_accessor :session 
+      
+    # Name of the left table
+    attr_accessor :left_table
+    
+    # Name of the right table
+    attr_accessor :right_table
+
+    # Cached array of primary key names
+    attr_accessor :primary_key_names
+
+    # Receives the active ScanProgressPrinters class
+    attr_accessor :progress_printer
+
+    # Returns a hash of scan options for this table scan.
+    def scan_options
+      @scan_options ||= session.configuration.options_for_table(left_table)
+    end
+
+    # Inform new progress to progress printer
+    # +steps+ is the number of processed records.
+    def update_progress(steps)
+      return unless progress_printer
+      unless @progress_printer_instance
+        total_records =
+          session.left.select_one("select count(*) as n from #{left_table}")['n'].to_i +
+          session.right.select_one("select count(*) as n from #{right_table}")['n'].to_i
+        @progress_printer_instance = progress_printer.new(total_records, session, left_table, right_table)
+      end
+      @progress_printer_instance.step(steps)
+    end
+    
+    # Creates a new DirectTableScan instance
+    #   * session: a Session object representing the current database session
+    #   * left_table: name of the table in the left database
+    #   * right_table: name of the table in the right database. If not given, same like left_table
+    def initialize(session, left_table, right_table = nil)
+      if session.left.primary_key_names(left_table).empty?
+        raise "Table '#{left_table}' doesn't have a primary key. Cannot scan."
+      end
+      
+      self.session, self.left_table, self.right_table = session, left_table, right_table
+      self.right_table ||= self.left_table
+      self.primary_key_names = session.left.primary_key_names left_table
+    end
+  end
+end

data/lib/rubyrep/table_scan_helper.rb

@@ -0,0 +1,38 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
+
+require 'rubyrep'
+
+module RR
+  
+  # Some helper functions that are of use to all TableScan classes
+  module TableScanHelper
+    # Compares the primary keys of left_row and right_row to determine their rank.
+    # Assumes there is a function primary_key_names returning the array of primary keys
+    # that are relevant for this comparison
+    # 
+    # Assumes that at least one of left_row and right_row is not nil
+    # A nil row counts as infinite. 
+    # E. g. left_row is something and right_row is nil ==> left_row is smaller ==> return -1
+    def rank_rows(left_row, right_row)
+      raise "At least one of left_row and right_row must not be nil!" unless left_row or right_row
+      return -1 unless right_row
+      return 1 unless left_row 
+      rank = 0
+      primary_key_names.any? do |key|
+        rank = left_row[key] <=> right_row[key]
+        rank != 0
+      end
+      rank
+    end
+
+    # Returns the correct class for the table scan based on the type of the
+    # session (proxied or direct).
+    def self.scan_class(session)
+      if session.proxied?
+        ProxiedTableScan
+      else
+        DirectTableScan
+      end
+    end
+  end
+end

data/lib/rubyrep/table_sorter.rb

@@ -0,0 +1,70 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
+
+require 'tsort'
+require 'rubyrep'
+
+module RR
+  # This class sorts a given list of tables so that tables referencing other
+  # tables via foreign keys are placed behind those referenced tables.
+  #
+  # Rationale:
+  # If tables are sorted in that sequence, the risk of foreign key violations is
+  # smaller.
+  class TableSorter
+    include TSort
+
+    # The active +Session+
+    attr_accessor :session
+
+    # The list of table names to be ordered
+    attr_accessor :tables
+
+    # The table dependencies.
+    # Format as described e. g. here: PostgreSQLExtender#referenced_tables
+    def referenced_tables
+      unless @referenced_tables
+        @referenced_tables = session.left.referenced_tables(tables)
+
+        # Strip away all unrelated tables
+        @referenced_tables.each_pair do |table, references|
+          references.delete_if do |reference|
+            not tables.include? reference
+          end
+        end
+      end
+      @referenced_tables
+    end
+
+    # Yields each table.
+    # For details see standard library: TSort#sort_each_node.
+    def tsort_each_node
+      referenced_tables.each_key do |table|
+        yield table
+      end
+    end
+
+    # Yields all tables that are references by +table+.
+    def tsort_each_child(table)
+      referenced_tables[table].each do |reference|
+        yield reference
+      end
+    end
+
+    def sort
+      # Note:
+      # We should not use TSort#tsort as this one throws an exception if
+      # there are cyclic redundancies.
+      # (Our goal is to just get the best ordering that is possible and then
+      # take our chances.)
+      strongly_connected_components.flatten
+    end
+
+    # Initializes the TableSorter
+    # * session: The active +Session+ instance
+    # * tables: an array of table names
+    def initialize(session, tables)
+      self.session = session
+      self.tables = tables
+    end
+  end
+end

data/lib/rubyrep/table_spec_resolver.rb

@@ -0,0 +1,136 @@
+module RR
+  
+  # Resolves table specifications as provided e. g. in the command line of rrscan
+  class TableSpecResolver
+    
+    # The +Session+ instance from which the table specifications are resolved.
+    attr_accessor :session
+
+    # Returns the array of tables of the specified database. Caches the table array.
+    # * database: either :+left+ or :+right+
+    def tables(database)
+      @table_cache ||= {}
+      unless @table_cache[database]
+        @table_cache[database] = session.send(database).tables
+      end
+      @table_cache[database]
+    end
+
+    # Creates a resolver that works based on the given +Session+ instance.
+    def initialize(session)
+      self.session = session
+    end
+
+    # Returns all those tables from the given table_pairs that do not exist.
+    # * +table_pairs+: same as described at #table_pairs_without_excluded
+    # 
+    # Returns:
+    # A hash with keys :+left+ and +:right+, with the value for each key being
+    # an array of non-existing tables for the according database.
+    # The keys only exist if there are according missing tables.
+    def non_existing_tables(table_pairs)
+      [:left, :right].inject({}) do |memo, database|
+        found_tables = table_pairs.inject([]) do |phantom_tables, table_pair|
+          phantom_tables << table_pair[database] unless tables(database).include?(table_pair[database])
+          phantom_tables
+        end
+        memo[database] = found_tables unless found_tables.empty?
+        memo
+      end
+    end
+    
+    # Resolves the given array of table specificifications.
+    # Table specifications are either
+    # * strings as produced by BaseRunner#get_options or
+    # * actual regular expressions
+    # If +excluded_table_specs+ is provided, removes all tables that match it
+    # (even if otherwise matching +included_table_specs+).
+    #
+    # If +verify+ is +true+, raises an exception if any non-existing tables are
+    # specified.
+    # 
+    # Returns an array of table name pairs in Hash form.
+    # For example something like
+    #   [{:left => 'my_table', :right => 'my_table_backup'}]
+    # 
+    # Takes care that a table is only returned once.
+    def resolve(included_table_specs, excluded_table_specs = [], verify = true)
+      table_pairs = expand_table_specs(included_table_specs)
+      table_pairs = table_pairs_without_duplicates(table_pairs)
+      table_pairs = table_pairs_without_excluded(table_pairs, excluded_table_specs)
+
+      if verify
+        non_existing_tables = non_existing_tables(table_pairs)
+        unless non_existing_tables.empty?
+          raise "non-existing tables specified: #{non_existing_tables.inspect}"
+        end
+      end
+
+      table_pairs
+    end
+
+    # Helper for #resolve
+    # Takes the specified table_specifications and expands it into an array of
+    # according table pairs.
+    # Returns the result
+    # Refer to #resolve for a full description of parameters and result.
+    def expand_table_specs(table_specs)
+      table_pairs = []
+      table_specs.each do |table_spec|
+
+        # If it is a regexp, convert it in an according string
+        table_spec = table_spec.inspect if table_spec.kind_of? Regexp
+
+        case table_spec
+        when /^\/.*\/$/ # matches e. g. '/^user/'
+          table_spec = table_spec.sub(/^\/(.*)\/$/,'\1') # remove leading and trailing slash
+          matching_tables = tables(:left).grep(Regexp.new(table_spec, Regexp::IGNORECASE, 'U'))
+          matching_tables.each do |table|
+            table_pairs << {:left => table, :right => table}
+          end
+        when /.+,.+/ # matches e. g. 'users,users_backup'
+          pair = table_spec.match(/(.*),(.*)/)[1..2].map { |str| str.strip }
+          table_pairs << {:left  => pair[0], :right => pair[1]}
+        else # everything else: just a normal table
+          table_pairs << {:left => table_spec.strip, :right => table_spec.strip}
+        end
+      end
+      table_pairs
+    end
+    private :expand_table_specs
+
+    # Helper for #resolve
+    # Takes given table_pairs and removes all tables that are excluded.
+    # Returns the result.
+    # Both the given and the returned table_pairs is an array of hashes with
+    # * :+left+: name of the left table
+    # * :+right+: name of the corresponding right table
+    # +excluded_table_specs+ is the array of table specifications to be excluded.
+    def table_pairs_without_excluded(table_pairs, excluded_table_specs)
+      excluded_tables = expand_table_specs(excluded_table_specs).map do |table_pair|
+        table_pair[:left]
+      end
+      table_pairs.select {|table_pair| not excluded_tables.include? table_pair[:left]}
+    end
+    private :table_pairs_without_excluded
+
+    # Helper for #resolve
+    # Takes given table_pairs and removes all duplicates.
+    # Returns the result.
+    # Both the given and the returned table_pairs is an array of hashes with
+    # * :+left+: name of the left table
+    # * :+right+: name of the corresponding right table
+    def table_pairs_without_duplicates(table_pairs)
+      processed_left_tables = {}
+      resulting_table_pairs = []
+      table_pairs.each do |table_pair|
+        unless processed_left_tables.include? table_pair[:left]
+          resulting_table_pairs << table_pair
+          processed_left_tables[table_pair[:left]] = true
+        end
+      end
+      resulting_table_pairs
+    end
+    private :table_pairs_without_duplicates
+  end
+end

data/lib/rubyrep/table_sync.rb

@@ -0,0 +1,68 @@
+module RR
+  
+  # Synchronizes the data of two tables.
+  class TableSync < TableScan
+
+    # Instance of SyncHelper
+    attr_accessor :helper
+
+    # Returns a hash of sync options for this table sync.
+    def sync_options
+      @sync_options ||= session.configuration.options_for_table(left_table)
+    end
+    
+    # Creates a new TableSync instance
+    #   * session: a Session object representing the current database session
+    #   * left_table: name of the table in the left database
+    #   * right_table: name of the table in the right database. If not given, same like left_table
+    def initialize(session, left_table, right_table = nil)
+      super
+    end
+
+    # Executes the specified sync hook
+    # * +hook_id+: either :+before_table_sync+ or :+after_table_sync+
+    def execute_sync_hook(hook_id)
+      hook = sync_options[hook_id]
+      if hook
+        if hook.respond_to?(:call)
+          hook.call(helper)
+        else
+          [:left, :right].each do |database|
+            session.send(database).execute hook
+          end
+        end
+      end
+    end
+
+    # Executes the table sync. If a block is given, yields each difference with
+    # the following 2 parameters
+    # * +type+
+    # * +row+
+    # Purpose: enable display of progress information.
+    # See DirectTableScan#run for full description of yielded parameters.
+    def run
+      success = false
+
+      scan_class = TableScanHelper.scan_class(session)
+      scan = scan_class.new(session, left_table, right_table)
+      scan.progress_printer = progress_printer
+
+      self.helper = SyncHelper.new(self)
+      syncer = Syncers.configured_syncer(sync_options).new(helper)
+    
+      execute_sync_hook :before_table_sync
+
+      scan.run do |type, row|
+        yield type, row if block_given? # To enable progress reporting
+        syncer.sync_difference type, row
+      end
+      
+      execute_sync_hook :after_table_sync
+
+      success = true # considered to be successful if we get till here
+    ensure
+      helper.finalize success if helper
+    end
+    
+  end
+end

data/lib/rubyrep/trigger_mode_switcher.rb

@@ -0,0 +1,63 @@
+require 'set'
+
+module RR
+
+  # Switches rubyrep triggers between "exclude rubyrep activity" modes.
+  class TriggerModeSwitcher
+
+    # Keeps track of all the triggers.
+    # This is a hash with 2 keys: :+left+ and :+right+.
+    # Each of these entries is a Set containing table names.
+    def triggers
+      @triggers ||= {
+        :left => Set.new,
+        :right => Set.new
+      }
+    end
+    
+    # The active Session
+    attr_accessor :session
+
+    def initialize(session)
+      self.session = session
+    end
+
+    # Does the actual switching of the trigger mode.
+    # * +database+: either :+left+ or :+right+
+    # * +table+: name of the table
+    # * +exclude_rr_activity+: the new trigger mode (either +true+ or +false+)
+    def switch_trigger_mode(database, table, exclude_rr_activity)
+      options = session.configuration.options
+      if session.send(database).replication_trigger_exists? "#{options[:rep_prefix]}_#{table}", table
+        params = {
+          :trigger_name => "#{options[:rep_prefix]}_#{table}",
+          :table => table,
+          :keys => session.send(database).primary_key_names(table),
+          :log_table => "#{options[:rep_prefix]}_pending_changes",
+          :activity_table => "#{options[:rep_prefix]}_running_flags",
+          :key_sep => options[:key_sep],
+          :exclude_rr_activity => exclude_rr_activity,
+        }
+        session.send(database).create_or_replace_replication_trigger_function(params)
+      end
+    end
+
+    # Switches the trigger of the named table to "exclude rubyrep activity" mode.
+    # Only switches if it didn't do so already for the table.
+    # * +database+: either :+left+ or :+right+
+    # * +table+: name of the table
+    def exclude_rr_activity(database, table)
+      switch_trigger_mode(database, table, true) if triggers[database].add? table
+    end
+
+    # Restores all switched triggers to not exclude rubyrep activity
+    def restore_triggers
+      [:left, :right].each do |database|
+        triggers[database].each do |table|
+          switch_trigger_mode database, table, false
+        end
+        triggers[database].clear
+      end
+    end
+  end
+end

data/lib/rubyrep/type_casting_cursor.rb

@@ -0,0 +1,31 @@
+module RR
+  # Provides functionality to cast a query result value into the correct ruby type.
+  # Requires originating table and column to be known.
+  class TypeCastingCursor
+
+    # Delegate the uninteresting methods to the original cursor
+    def next?; org_cursor.next? end
+    def clear; org_cursor.clear end
+    
+    # The original cursor object
+    attr_accessor :org_cursor
+    
+    # A column_name => Column cache
+    attr_accessor :columns
+    
+    # Creates a new TypeCastingCursor based on provided database connection and table name
+    # for the provided database query cursor
+    def initialize(connection, table, cursor)
+      self.org_cursor = cursor
+      self.columns = {}
+      connection.columns(table).each {|c| columns[c.name] = c}
+    end
+    
+    # Reads the next row from the original cursor and returns the row with the type casted row values.
+    def next_row
+      row = org_cursor.next_row
+      row.each {|column, value| row[column] = columns[column].type_cast value}
+      row
+    end    
+  end
+end