rubyrep 1.0.0

data/lib/rubyrep/scan_report_printers/scan_summary_reporter.rb

@@ -0,0 +1,75 @@
+module RR::ScanReportPrinters
+  # A ScanReportPrinter producing a summary (number of differences) only.
+  class ScanSummaryReporter
+    
+    # Register ScanSummaryReporter with the given command line options.
+    # (Command line format as specified by OptionParser#on.)
+    RR::ScanReportPrinters.register self, "-s", "--summary[=detailed]",
+        "Print the number of differences of each table. Either totals only, e. g.",
+        "  left_table / right_table [differences]",
+        "or a detailed split by type, e. g.",
+        "  left_table / right_table [conflicts] [left_only records] [right_only records]"
+    
+    # Set to true if only the total number of differences should be reported
+    attr_accessor :only_totals
+    
+    # Name of the left table of the current scan
+    attr_accessor :left_table
+    
+    # Name of the right table of the current scan
+    attr_accessor :right_table
+    
+    # Hold the result of the current scan. A hash with a running count of
+    #  +:conflict+, +:left+ (only) or +:right+ (only) records.
+    attr_accessor :scan_result
+
+    # A scan run is to be started using this scan result printer.
+    # +arg+ is the command line argument as yielded by OptionParser#on.
+    def initialize(_, arg)
+      self.only_totals = (arg != 'detailed')
+    end
+    
+    # A scan of the given 'left' table and corresponding 'right' table is executed.
+    # Needs to yield so that the actual scan can be executed.
+    def scan(left_table, right_table)
+      self.left_table = left_table
+      self.right_table = right_table
+      self.scan_result = {:conflict => 0, :left => 0, :right => 0}
+
+      header = left_table.clone
+      header << " / " << right_table if left_table != right_table
+      $stdout.write "#{header.rjust(36)} "
+
+      yield # Give control back so that the actual table scan can be done.
+
+      if only_totals
+        $stdout.write \
+          "#{rjust_value(scan_result[:conflict] + scan_result[:left] + scan_result[:right])}"
+      else
+        $stdout.write \
+          "#{rjust_value(scan_result[:conflict])} " +
+          "#{rjust_value(scan_result[:left])} " +
+          "#{rjust_value(scan_result[:right])}"
+      end
+      $stdout.puts
+    end
+
+    # Right adjusts the given number and returns according string.
+    def rjust_value(value)
+      value.to_s.rjust(3)
+    end
+    private :rjust_value
+
+    # Each difference is handed to the printer as described in the format
+    # as described e. g. in DirectTableScan#run
+    def report_difference(type, row)
+      scan_result[type] += 1
+    end
+
+    # Optional method. If a scan report printer has it, it is called after the
+    # last table scan is executed.
+    # (A good place to print a final summary.)
+    def scanning_finished
+    end
+  end
+end

data/lib/rubyrep/scan_runner.rb

@@ -0,0 +1,25 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+
+module RR
+  # This class implements the functionality of the rrscan.rb command.
+  class ScanRunner < BaseRunner
+
+    CommandRunner.register 'scan' => {
+      :command => self,
+      :description => 'Scans for differing records between databases'
+    }
+
+    # Returns summary description string for the scan command.
+    def summary_description
+      "Scans for differences of the specified tables between both databases."
+    end
+    
+    # Creates the correct scan class.
+    # Parameters as defined under BaseRunner#create_processor
+    def create_processor(left_table, right_table)
+      TableScanHelper.scan_class(session).new session, left_table, right_table
+    end
+  end
+end
+
+

data/lib/rubyrep/session.rb

@@ -0,0 +1,177 @@
+require 'drb'
+
+module RR
+
+  # This class represents a rubyrep session.
+  # Creates and holds expensive objects like e. g. database connections.
+  class Session
+    
+    # The Configuration object provided to the initializer
+    attr_accessor :configuration
+    
+    # Returns the "left" ActiveRecord / proxy database connection
+    def left
+      @connections[:left]
+    end
+    
+    # Stores the "left" ActiveRecord /proxy database connection
+    def left=(connection)
+      @connections[:left] = connection
+    end
+    
+    # Returns the "right" ActiveRecord / proxy database connection
+    def right
+      @connections[:right]
+    end
+    
+    # Stores the "right" ActiveRecord / proxy database connection
+    def right=(connection)
+      @connections[:right] = connection
+    end
+    
+    # Hash to hold under either :left or :right the according Drb / direct DatabaseProxy
+    attr_accessor :proxies
+
+    # Creates a hash of manual primary key names as can be specified with the
+    # Configuration options :+primary_key_names+ or :+auto_key_limit+.
+    # * +db_arm: should be either :left or :right
+    #
+    # Returns the identified manual primary keys. This is a hash with
+    # * key: table_name
+    # * value: array of primary key names
+    def manual_primary_keys(db_arm)
+      manual_primary_keys = {}
+      resolver = TableSpecResolver.new self
+      table_pairs = resolver.resolve configuration.included_table_specs, [], false
+      table_pairs.each do |table_pair|
+        options = configuration.options_for_table(table_pair[:left])
+        key_names = options[:key]
+        if key_names == nil and options[:auto_key_limit] > 0
+          if left.primary_key_names(table_pair[:left], :raw => true).empty?
+            column_names = left.column_names(table_pair[:left])
+            if column_names.size <= options[:auto_key_limit]
+              key_names = column_names
+            end
+          end
+        end
+        if key_names
+          table_name = table_pair[db_arm]
+          manual_primary_keys[table_name] = [key_names].flatten
+        end
+      end
+      manual_primary_keys
+    end
+
+    # Returns the corresponding table in the other database.
+    # * +db_arm+: database of the given table (either :+left+ or :+right+)
+    # * +table+: name of the table
+    #
+    # If no corresponding table can be found, return the given table.
+    # Rationale:
+    # Support the case where a table was dropped from the configuration but
+    # there were still some unreplicated changes left.
+    def corresponding_table(db_arm, table)
+      unless @table_map
+        @table_map = {:left => {}, :right => {}}
+        resolver = TableSpecResolver.new self
+        table_pairs = resolver.resolve configuration.included_table_specs, [], false
+        table_pairs.each do |table_pair|
+          @table_map[:left][table_pair[:left]] = table_pair[:right]
+          @table_map[:right][table_pair[:right]] = table_pair[:left]
+        end
+      end
+      @table_map[db_arm][table] || table
+    end
+    
+    # Does the actual work of establishing a database connection
+    # db_arm:: should be either :left or :right
+    # config:: the rubyrep Configuration
+    def db_connect(db_arm, config)
+      arm_config = config.send db_arm
+      @proxies[db_arm] = DatabaseProxy.new
+      @connections[db_arm] = @proxies[db_arm].create_session arm_config
+    end
+    
+    # Does the actual work of establishing a proxy connection
+    # db_arm:: should be either :left or :right
+    # config:: the rubyrep Configuration
+    def proxy_connect(db_arm, config)
+      arm_config = config.send db_arm
+      if arm_config.include? :proxy_host 
+        drb_url = "druby://#{arm_config[:proxy_host]}:#{arm_config[:proxy_port]}"
+        @proxies[db_arm] = DRbObject.new nil, drb_url
+      else
+        # If one connection goes through a proxy, so has the other one.
+        # So if necessary, create a "fake" proxy
+        @proxies[db_arm] = DatabaseProxy.new
+      end
+      @connections[db_arm] = @proxies[db_arm].create_session arm_config
+    end
+    
+    # True if proxy connections are used
+    def proxied?
+      [configuration.left, configuration.right].any? \
+        {|arm_config| arm_config.include? :proxy_host}
+    end
+
+    # Returns an array of table pairs of the configured tables.
+    # Refer to TableSpecResolver#resolve for a detailed description of the
+    # return value.
+    # If +included_table_specs+ is provided (that is: not an empty array), it
+    # will be used instead of the configured table specs.
+    def configured_table_pairs(included_table_specs = [])
+      resolver = TableSpecResolver.new self
+      included_table_specs = configuration.included_table_specs if included_table_specs.empty?
+      resolver.resolve included_table_specs, configuration.excluded_table_specs
+    end
+
+    # Orders the array of table pairs as per primary key / foreign key relations
+    # of the tables. Returns the result.
+    # Only sorts if the configuration has set option :+table_ordering+.
+    # Refer to TableSpecResolver#resolve for a detailed description of the
+    # parameter and return value.
+    def sort_table_pairs(table_pairs)
+      if configuration.options[:table_ordering]
+        left_tables = table_pairs.map {|table_pair| table_pair[:left]}
+        sorted_left_tables = TableSorter.new(self, left_tables).sort
+        sorted_left_tables.map do |left_table|
+          table_pairs.find do |table_pair|
+            table_pair[:left] == left_table
+          end
+        end
+      else
+        table_pairs
+      end
+    end
+
+    # Refreshes (reestablish if no more active) the database connections.
+    def refresh
+      [:left, :right].each {|database| send(database).refresh}
+    end
+        
+    # Creates a new rubyrep session with the provided Configuration
+    def initialize(config = Initializer::configuration)
+      @connections = {:left => nil, :right => nil}
+      @proxies = {:left => nil, :right => nil}
+      
+      # Keep the database configuration for future reference
+      self.configuration = config
+
+      # Determine method of connection (either 'proxy_connect' or 'db_connect'
+      connection_method = proxied? ? :proxy_connect : :db_connect
+      
+      # Connect the left database / proxy
+      self.send connection_method, :left, configuration
+      left.manual_primary_keys = manual_primary_keys(:left)
+      
+      # If both database configurations point to the same database
+      # then don't create the database connection twice
+      if configuration.left == configuration.right
+        self.right = self.left
+      else
+        self.send connection_method, :right, configuration
+        right.manual_primary_keys = manual_primary_keys(:right)
+      end  
+    end
+  end
+end

data/lib/rubyrep/sync_helper.rb

@@ -0,0 +1,111 @@
+module RR
+
+  # Provides helper functionality for the table syncers.
+  # The methods exposed by this class are intended to provide a stable interface
+  # for third party syncers.
+  class SyncHelper
+
+    # The current +TableSync+ instance
+    attr_accessor :table_sync
+
+    # The active +Session+
+    def session; table_sync.session; end
+
+    # Name of the left table
+    def left_table; table_sync.left_table; end
+
+    # Name of the right table
+    def right_table; table_sync.right_table; end
+
+    # A hash with
+    # :+left+: name of the table in the left database
+    # :+right+: name of the table in the right database
+    def tables
+      @tables ||= {:left => left_table, :right => right_table}
+    end
+
+    # Sync options for the current table sync
+    def sync_options; @sync_options ||= table_sync.sync_options; end
+
+    # Delegates to Committer#insert_record
+    def insert_record(database, values)
+      committer.insert_record(database, tables[database], values)
+    end
+
+    # Delegates to Committer#insert_record
+    def update_record(database, values, old_key = nil)
+      committer.update_record(database, tables[database], values, old_key)
+    end
+
+    # Delegates to Committer#insert_record
+    def delete_record(database, values)
+      committer.delete_record(database, tables[database], values)
+    end
+
+    # Return the committer, creating it if not yet there.
+    def committer
+      unless @committer
+        committer_class = Committers::committers[sync_options[:committer]]
+        @committer = committer_class.new(session)
+      end
+      @committer
+    end
+    private :committer
+
+    # Checks if the event log table already exists and creates it if necessary
+    def ensure_event_log
+      unless @ensured_event_log
+        ReplicationInitializer.new(session).ensure_event_log
+        @ensured_event_log = true
+      end
+    end
+
+    # Returns an array of primary key names for the synced table
+    def primary_key_names
+      @primary_key_names ||= session.left.primary_key_names(left_table)
+    end
+    private :primary_key_names
+
+    # Logs the outcome of a replication into the replication log table.
+    # * +row+: a column_name => value hash for at least the primary keys of the record
+    # * +type+: string describing the type of the sync
+    # * +outcome+: string describing what's done about the sync
+    # * +details+: string with further details regarding the sync
+    def log_sync_outcome(row, type, outcome, details = nil)
+      ensure_event_log
+      if primary_key_names.size == 1
+        key = row[primary_key_names[0]]
+      else
+        key_parts = primary_key_names.map do |column_name|
+          %Q("#{column_name}"=>#{row[column_name].to_s.inspect})
+        end
+        key = key_parts.join(', ')
+      end
+      sync_details = details == nil ? nil : details[0...ReplicationInitializer::LONG_DESCRIPTION_SIZE]
+
+      session.left.insert_record "#{sync_options[:rep_prefix]}_logged_events", {
+        :activity => 'sync',
+        :change_table => left_table,
+        :diff_type => type.to_s,
+        :change_key => key,
+        :left_change_type => nil,
+        :right_change_type => nil,
+        :description => outcome.to_s,
+        :long_description => sync_details,
+        :event_time => Time.now,
+        :diff_dump => nil
+      }
+    end
+
+    # Asks the committer (if it exists) to finalize any open transactions
+    # +success+ should be true if there were no problems, false otherwise.
+    def finalize(success = true)
+      @committer.finalize(success) if @committer
+    end
+    
+    # Creates a new SyncHelper for the given +TableSync+ instance.
+    def initialize(table_sync)
+      self.table_sync = table_sync
+    end
+  end
+end

data/lib/rubyrep/sync_runner.rb

@@ -0,0 +1,31 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+
+module RR
+  # This class implements the functionality of the rrsync.rb command.
+  class SyncRunner < BaseRunner
+
+    CommandRunner.register 'sync' => {
+      :command => self,
+      :description => 'Syncs records between databases'
+    }
+
+    # Returns summary description string for the scan command.
+    def summary_description
+      "Syncs the differences of the specified tables between both databases."
+    end
+
+    # Creates the correct scan class.
+    # Parameters as defined under BaseRunner#create_processor
+    def create_processor(left_table, right_table)
+      TableSync.new session, left_table, right_table
+    end
+
+    # Reorders the table pairs to avoid foreign key conflicts.
+    # More information on this methods at BaseRunner#prepare_table_pairs.
+    def prepare_table_pairs(table_pairs)
+      session.sort_table_pairs(table_pairs)
+    end
+  end
+end
+
+

data/lib/rubyrep/syncers/syncers.rb

@@ -0,0 +1,112 @@
+module RR
+  # Syncers are classes that implement the sync policies.
+  # This module provides functionality to register syncers and access the
+  # list of registered syncers.
+  # Each Syncer must register itself with Syncers#register.
+  # Each Syncer must implement the following methods:
+  #
+  #   # Creates a new syncer (A syncer is used for one table sync only)
+  #   #   * sync_helper: a SyncHelper object providing necessary information and functionalities
+  #   def initialize(sync_helper)
+  #
+  #   # Called to sync the provided difference.
+  #   # See DirectTableScan#run for a description of the +type+ and +row+ parameters.
+  #   def sync_difference(type, row)
+  #
+  #   # Provides default option for the syncer. Optional.
+  #   # Returns a hash with :key => value pairs.
+  #   def self.default_options
+  module Syncers
+    # Returns a Hash of currently registered syncers.
+    # (Empty Hash if no syncers were defined.)
+    def self.syncers
+      @syncers ||= {}
+      @syncers
+    end
+  
+    # Registers one or multiple syncers.
+    # syncer_hash is a Hash with
+    #   key::   The adapter symbol as used to reference the syncer
+    #   value:: The class implementing the syncer
+    def self.register(syncer_hash)
+      @syncers ||= {}
+      @syncers.merge! syncer_hash
+    end
+
+    # Returns the correct syncer class as per provided options hash
+    def self.configured_syncer(options)
+      syncer_id = options[:syncer]
+      syncer_id ||= options[:replicator]
+      syncers[syncer_id]
+    end
+    
+    # This syncer implements a one way sync.
+    # Syncer options relevant for this syncer:
+    #   * +:direction+: Sync direction. Possible values:
+    #     * +:left+
+    #     * +:right+
+    #   * +:delete+: Default: false. If true, deletes in the target database all
+    #                records _not_ existing in the source database.
+    #   * +:update+: If true (default), update records in the target database
+    #                if different.
+    #   * +:insert+: If true (default), copy over records not existing in the
+    #                target database.
+    class OneWaySyncer
+      
+      # Register the syncer
+      Syncers.register :one_way => self
+
+      # The current SyncHelper object
+      attr_accessor :sync_helper
+
+      # ID of source database (either :left or :right)
+      attr_accessor :source
+
+      # ID of target database (either :left or :right)
+      attr_accessor :target
+
+      # Array index to source row in case #sync_difference +type+ is :conflict.
+      # (As in that case the +row+ parameter is an array of left and right records.)
+      attr_accessor :source_record_index
+      
+      # Provides default option for the syncer. Optional.
+      # Returns a hash with :key => value pairs.
+      def self.default_options
+        {
+          :direction => :right,
+          :delete => false, :update => true, :insert => true
+        }
+      end
+
+      # Initializes the syncer
+      #   * sync_helper: The SyncHelper object provided information and utility
+      #                  functions.
+      def initialize(sync_helper)
+        self.sync_helper = sync_helper
+        self.source = sync_helper.sync_options[:direction] == :left ? :right : :left
+        self.target = sync_helper.sync_options[:direction] == :left ? :left : :right
+        self.source_record_index = sync_helper.sync_options[:direction] == :left ? 1 : 0
+      end
+
+      # Called to sync the provided difference.
+      # See DirectTableScan#run for a description of the +type+ and +row+ parameters.
+      def sync_difference(type, row)
+        case type
+        when source
+          if sync_helper.sync_options[:insert]
+            sync_helper.insert_record target, row
+          end
+        when target
+          if sync_helper.sync_options[:delete]
+            sync_helper.delete_record target, row
+          end
+        when :conflict
+          if sync_helper.sync_options[:update]
+            sync_helper.update_record target, row[source_record_index]
+          end
+        end
+      end
+    end
+    
+  end
+end