andyjeffries-rubyrep 1.2.1

data/lib/rubyrep/table_scan_helper.rb

@@ -0,0 +1,46 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
+
+require 'rubyrep'
+
+module RR
+  
+  # Some helper functions that are of use to all TableScan classes
+  module TableScanHelper
+    # Compares the primary keys of left_row and right_row to determine their rank.
+    # Assumes there is a function primary_key_names returning the array of primary keys
+    # that are relevant for this comparison
+    # 
+    # Assumes that at least one of left_row and right_row is not nil
+    # A nil row counts as infinite. 
+    # E. g. left_row is something and right_row is nil ==> left_row is smaller ==> return -1
+    def rank_rows(left_row, right_row)
+      raise "At least one of left_row and right_row must not be nil!" unless left_row or right_row
+      return -1 unless right_row
+      return 1 unless left_row 
+      rank = 0
+      primary_key_names.any? do |key|
+        if left_row[key].kind_of?(String)
+          # When databases order strings, then 'a' < 'A' while for Ruby 'A' < 'a'
+          # ==> Use a combination of case sensitive and case insensitive comparing to
+          #     reproduce the database behaviour.
+          rank = left_row[key].casecmp(right_row[key]) # deal with 'a' to 'B' comparisons
+          rank = -(left_row[key] <=> right_row[key]) if rank == 0 # deal with 'a' to 'A' comparisons
+        else
+          rank = left_row[key] <=> right_row[key]
+        end
+        rank != 0
+      end
+      rank
+    end
+
+    # Returns the correct class for the table scan based on the type of the
+    # session (proxied or direct).
+    def self.scan_class(session)
+      if session.proxied?
+        ProxiedTableScan
+      else
+        DirectTableScan
+      end
+    end
+  end
+end

data/lib/rubyrep/table_sorter.rb

@@ -0,0 +1,70 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
+
+require 'tsort'
+require 'rubyrep'
+
+module RR
+  # This class sorts a given list of tables so that tables referencing other
+  # tables via foreign keys are placed behind those referenced tables.
+  #
+  # Rationale:
+  # If tables are sorted in that sequence, the risk of foreign key violations is
+  # smaller.
+  class TableSorter
+    include TSort
+
+    # The active +Session+
+    attr_accessor :session
+
+    # The list of table names to be ordered
+    attr_accessor :tables
+
+    # The table dependencies.
+    # Format as described e. g. here: PostgreSQLExtender#referenced_tables
+    def referenced_tables
+      unless @referenced_tables
+        @referenced_tables = session.left.referenced_tables(tables)
+
+        # Strip away all unrelated tables
+        @referenced_tables.each_pair do |table, references|
+          references.delete_if do |reference|
+            not tables.include? reference
+          end
+        end
+      end
+      @referenced_tables
+    end
+
+    # Yields each table.
+    # For details see standard library: TSort#sort_each_node.
+    def tsort_each_node
+      referenced_tables.each_key do |table|
+        yield table
+      end
+    end
+
+    # Yields all tables that are references by +table+.
+    def tsort_each_child(table)
+      referenced_tables[table].each do |reference|
+        yield reference
+      end
+    end
+
+    def sort
+      # Note:
+      # We should not use TSort#tsort as this one throws an exception if
+      # there are cyclic redundancies.
+      # (Our goal is to just get the best ordering that is possible and then
+      # take our chances.)
+      strongly_connected_components.flatten
+    end
+
+    # Initializes the TableSorter
+    # * session: The active +Session+ instance
+    # * tables: an array of table names
+    def initialize(session, tables)
+      self.session = session
+      self.tables = tables
+    end
+  end
+end

data/lib/rubyrep/table_spec_resolver.rb

@@ -0,0 +1,142 @@
+module RR
+  
+  # Resolves table specifications as provided e. g. in the command line of rrscan
+  class TableSpecResolver
+    
+    # The +Session+ instance from which the table specifications are resolved.
+    attr_accessor :session
+
+    # Returns the array of tables of the specified database. Caches the table array.
+    # * database: either :+left+ or :+right+
+    def tables(database)
+      @table_cache ||= {}
+      unless @table_cache[database]
+        @table_cache[database] = session.send(database).tables
+      end
+      @table_cache[database]
+    end
+
+    # Creates a resolver that works based on the given +Session+ instance.
+    def initialize(session)
+      self.session = session
+    end
+
+    # Returns all those tables from the given table_pairs that do not exist.
+    # * +table_pairs+: same as described at #table_pairs_without_excluded
+    # 
+    # Returns:
+    # A hash with keys :+left+ and +:right+, with the value for each key being
+    # an array of non-existing tables for the according database.
+    # The keys only exist if there are according missing tables.
+    def non_existing_tables(table_pairs)
+      [:left, :right].inject({}) do |memo, database|
+        found_tables = table_pairs.inject([]) do |phantom_tables, table_pair|
+          phantom_tables << table_pair[database] unless tables(database).include?(table_pair[database])
+          phantom_tables
+        end
+        memo[database] = found_tables unless found_tables.empty?
+        memo
+      end
+    end
+    
+    # Resolves the given array of table specificifications.
+    # Table specifications are either
+    # * strings as produced by BaseRunner#get_options or
+    # * actual regular expressions
+    # If +excluded_table_specs+ is provided, removes all tables that match it
+    # (even if otherwise matching +included_table_specs+).
+    #
+    # If +verify+ is +true+, raises an exception if any non-existing tables are
+    # specified.
+    # 
+    # Returns an array of table name pairs in Hash form.
+    # For example something like
+    #   [{:left => 'my_table', :right => 'my_table_backup'}]
+    # 
+    # Takes care that a table is only returned once.
+    def resolve(included_table_specs, excluded_table_specs = [], verify = true)
+      table_pairs = expand_table_specs(included_table_specs, verify)
+      table_pairs = table_pairs_without_duplicates(table_pairs)
+      table_pairs = table_pairs_without_excluded(table_pairs, excluded_table_specs)
+
+      if verify
+        non_existing_tables = non_existing_tables(table_pairs)
+        unless non_existing_tables.empty?
+          raise "non-existing tables specified: #{non_existing_tables.inspect}"
+        end
+      end
+
+      table_pairs
+    end
+
+    # Helper for #resolve
+    # Expands table specifications into table pairs.
+    # Parameters:
+    # * +table_specs+:
+    #   An array of table specifications as described under #resolve.
+    # * +verify+:
+    #   If +true+, table specs in regexp format only resolve if the table exists
+    #   in left and right database.
+    # Return value: refer to #resolve for a detailed description
+    def expand_table_specs(table_specs, verify)
+      table_pairs = []
+      table_specs.each do |table_spec|
+
+        # If it is a regexp, convert it in an according string
+        table_spec = table_spec.inspect if table_spec.kind_of? Regexp
+
+        case table_spec
+        when /^\/.*\/$/ # matches e. g. '/^user/'
+          table_spec = table_spec.sub(/^\/(.*)\/$/,'\1') # remove leading and trailing slash
+          matching_tables = tables(:left).grep(Regexp.new(table_spec, Regexp::IGNORECASE))
+          matching_tables.each do |table|
+            if !verify or tables(:right).include? table
+              table_pairs << {:left => table, :right => table}
+            end
+          end
+        when /.+,.+/ # matches e. g. 'users,users_backup'
+          pair = table_spec.match(/(.*),(.*)/)[1..2].map { |str| str.strip }
+          table_pairs << {:left  => pair[0], :right => pair[1]}
+        else # everything else: just a normal table
+          table_pairs << {:left => table_spec.strip, :right => table_spec.strip}
+        end
+      end
+      table_pairs
+    end
+    private :expand_table_specs
+
+    # Helper for #resolve
+    # Takes given table_pairs and removes all tables that are excluded.
+    # Returns the result.
+    # Both the given and the returned table_pairs is an array of hashes with
+    # * :+left+: name of the left table
+    # * :+right+: name of the corresponding right table
+    # +excluded_table_specs+ is the array of table specifications to be excluded.
+    def table_pairs_without_excluded(table_pairs, excluded_table_specs)
+      excluded_tables = expand_table_specs(excluded_table_specs, false).map do |table_pair|
+        table_pair[:left]
+      end
+      table_pairs.select {|table_pair| not excluded_tables.include? table_pair[:left]}
+    end
+    private :table_pairs_without_excluded
+
+    # Helper for #resolve
+    # Takes given table_pairs and removes all duplicates.
+    # Returns the result.
+    # Both the given and the returned table_pairs is an array of hashes with
+    # * :+left+: name of the left table
+    # * :+right+: name of the corresponding right table
+    def table_pairs_without_duplicates(table_pairs)
+      processed_left_tables = {}
+      resulting_table_pairs = []
+      table_pairs.each do |table_pair|
+        unless processed_left_tables.include? table_pair[:left]
+          resulting_table_pairs << table_pair
+          processed_left_tables[table_pair[:left]] = true
+        end
+      end
+      resulting_table_pairs
+    end
+    private :table_pairs_without_duplicates
+  end
+end

data/lib/rubyrep/table_sync.rb

@@ -0,0 +1,90 @@
+module RR
+  
+  # Synchronizes the data of two tables.
+  class TableSync < TableScan
+
+    # Instance of SyncHelper
+    attr_accessor :helper
+
+    # Returns a hash of sync options for this table sync.
+    def sync_options
+      @sync_options ||= session.configuration.options_for_table(left_table)
+    end
+    
+    # Creates a new TableSync instance
+    #   * session: a Session object representing the current database session
+    #   * left_table: name of the table in the left database
+    #   * right_table: name of the table in the right database. If not given, same like left_table
+    def initialize(session, left_table, right_table = nil)
+      super
+    end
+
+    # Executes the specified sync hook
+    # * +hook_id+: either :+before_table_sync+ or :+after_table_sync+
+    def execute_sync_hook(hook_id)
+      hook = sync_options[hook_id]
+      if hook
+        if hook.respond_to?(:call)
+          hook.call(helper)
+        else
+          [:left, :right].each do |database|
+            session.send(database).execute hook
+          end
+        end
+      end
+    end
+
+    # Calls the event filter for the give table difference.
+    # * +type+: type of difference
+    # * +row+: the differing row
+    # Refer to DirectTableScan#run for full description of +type+ and +row+.
+    # Returns +true+ if syncing of the difference should *not* proceed.
+    def event_filtered?(type, row)
+      event_filter = sync_options[:event_filter]
+      if event_filter && event_filter.respond_to?(:before_sync)
+        not event_filter.before_sync(
+          helper.left_table,
+          helper.extract_key([row].flatten.first),
+          helper,
+          type,
+          row
+        )
+      else
+        false
+      end
+    end
+
+    # Executes the table sync. If a block is given, yields each difference with
+    # the following 2 parameters
+    # * +type+
+    # * +row+
+    # Purpose: enable display of progress information.
+    # See DirectTableScan#run for full description of yielded parameters.
+    def run
+      success = false
+
+      scan_class = TableScanHelper.scan_class(session)
+      scan = scan_class.new(session, left_table, right_table)
+      scan.progress_printer = progress_printer
+
+      self.helper = SyncHelper.new(self)
+      syncer = Syncers.configured_syncer(sync_options).new(helper)
+    
+      execute_sync_hook :before_table_sync
+
+      scan.run do |type, row|
+        yield type, row if block_given? # To enable progress reporting
+        unless event_filtered?(type, row)
+          syncer.sync_difference type, row
+        end
+      end
+      
+      execute_sync_hook :after_table_sync
+
+      success = true # considered to be successful if we get till here
+    ensure
+      helper.finalize success if helper
+    end
+    
+  end
+end

data/lib/rubyrep/task_sweeper.rb

@@ -0,0 +1,77 @@
+module RR
+
+  # Monitors and cancels stalled tasks
+  class TaskSweeper
+
+    # Executes the give block in a separate Thread.
+    # Returns if block is finished or stalled.
+    # The block must call regular #ping to announce it is not stalled.
+    # * +timeout_period+:
+    #   Maximum time (in seonds) without ping, after which a task is considered stalled.
+    # Returns the created sweeper (allows checking if task was terminated).
+    def self.timeout(timeout_period)
+      sweeper = TaskSweeper.new(timeout_period)
+      sweeper.send(:timeout) {yield sweeper}
+      sweeper
+    end
+
+    # Time in seconds after which a task is considered stalled. Timer is reset
+    # by calling #ping.
+    attr_accessor :timeout_period
+
+    # Must be called by the executed task to announce it is still alive
+    def ping
+      self.last_ping = Time.now
+    end
+
+    # Returns +true+ if the task was timed out.
+    # The terminated task is expected to free all resources and exit.
+    def terminated?
+      terminated
+    end
+
+    # Waits without timeout till the task executing thread is finished
+    def join
+      thread && thread.join
+    end
+
+    # Creates a new TaskSweeper
+    # * +timeout_period+: timeout value in seconds
+    def initialize(timeout_period)
+      self.timeout_period = timeout_period
+      self.terminated = false
+      self.last_ping = Time.now
+    end
+
+    protected
+
+    # Time of last ping
+    attr_accessor :last_ping
+
+    # Set to +true+ if the executed task has timed out
+    attr_accessor :terminated
+
+    # The task executing thread
+    attr_accessor :thread
+
+    # Executes the given block and times it out if stalled.
+    def timeout
+      exception = nil
+      self.thread = Thread.new do
+        begin
+          yield
+        rescue Exception => e
+          # save exception so it can be rethrown outside of the thread
+          exception = e
+        end
+      end
+      while self.thread.join(self.timeout_period) == nil do
+        if self.last_ping < Time.now - self.timeout_period
+          self.terminated = true
+          break
+        end
+      end
+      raise exception if exception
+    end
+  end
+end

data/lib/rubyrep/trigger_mode_switcher.rb

@@ -0,0 +1,63 @@
+require 'set'
+
+module RR
+
+  # Switches rubyrep triggers between "exclude rubyrep activity" modes.
+  class TriggerModeSwitcher
+
+    # Keeps track of all the triggers.
+    # This is a hash with 2 keys: :+left+ and :+right+.
+    # Each of these entries is a Set containing table names.
+    def triggers
+      @triggers ||= {
+        :left => Set.new,
+        :right => Set.new
+      }
+    end
+    
+    # The active Session
+    attr_accessor :session
+
+    def initialize(session)
+      self.session = session
+    end
+
+    # Does the actual switching of the trigger mode.
+    # * +database+: either :+left+ or :+right+
+    # * +table+: name of the table
+    # * +exclude_rr_activity+: the new trigger mode (either +true+ or +false+)
+    def switch_trigger_mode(database, table, exclude_rr_activity)
+      options = session.configuration.options
+      if session.send(database).replication_trigger_exists? "#{options[:rep_prefix]}_#{table}", table
+        params = {
+          :trigger_name => "#{options[:rep_prefix]}_#{table}",
+          :table => table,
+          :keys => session.send(database).primary_key_names(table),
+          :log_table => "#{options[:rep_prefix]}_pending_changes",
+          :activity_table => "#{options[:rep_prefix]}_running_flags",
+          :key_sep => options[:key_sep],
+          :exclude_rr_activity => exclude_rr_activity,
+        }
+        session.send(database).create_or_replace_replication_trigger_function(params)
+      end
+    end
+
+    # Switches the trigger of the named table to "exclude rubyrep activity" mode.
+    # Only switches if it didn't do so already for the table.
+    # * +database+: either :+left+ or :+right+
+    # * +table+: name of the table
+    def exclude_rr_activity(database, table)
+      switch_trigger_mode(database, table, true) if triggers[database].add? table
+    end
+
+    # Restores all switched triggers to not exclude rubyrep activity
+    def restore_triggers
+      [:left, :right].each do |database|
+        triggers[database].each do |table|
+          switch_trigger_mode database, table, false
+        end
+        triggers[database].clear
+      end
+    end
+  end
+end