andyjeffries-rubyrep 1.2.1

data/lib/rubyrep/scan_progress_printers/progress_bar.rb

@@ -0,0 +1,65 @@
+module RR
+  module ScanProgressPrinters
+    
+    # A helper class to print a text progress bar.
+    class ProgressBar
+
+      MAX_MARKERS = 25 #length of the progress bar (in characters)
+
+      # Register ProgressBar with the given command line options.
+      # (Command line format as specified by OptionParser#on.)
+      # First argument is the key through which the printer can be refered in
+      # the configuration file
+      RR::ScanProgressPrinters.register :progress_bar, self,
+        "-b", "--progress-bar[=length]",
+        "Show the progress of the table scanning process as progress bar."
+
+      # Receives the command line argument
+      cattr_accessor :arg
+
+      # Returns the length (in characters) of the progress bar.
+      def max_markers
+        @max_markers ||= arg ? arg.to_i : MAX_MARKERS
+      end
+
+      # Creates a new progress bar.
+      # * +max_steps+: number of steps at completion
+      # * +session+: the current Session
+      # * +left_table+: name of the left database table
+      # * +right_table+: name of the right database table
+      def initialize(max_steps, session, left_table, right_table)
+        @use_ansi = session.configuration.options_for_table(left_table)[:use_ansi]
+        @max_steps, @current_steps = max_steps, 0
+        @steps_per_marker = @max_steps.to_f / max_markers
+        @current_markers, @current_percentage = 0, 0
+      end
+  
+      # Increases progress by +step_increment+ steps.
+      def step(step_increment = 1)
+        @current_steps+= step_increment
+        new_markers = @max_steps != 0 ? (@current_steps / @steps_per_marker).to_i : max_markers
+
+        new_percentage = @max_steps != 0 ? @current_steps * 100 / @max_steps : 100
+        if @use_ansi and new_percentage != @current_percentage
+          # This part uses ANSI escape sequences to show a running percentage
+          # to the left of the progress bar
+          print "\e[1D" * (@current_markers + 5) if @current_percentage != 0 # go left
+          print "#{new_percentage}%".rjust(4) << " "
+          print "\e[1C" * @current_markers if @current_markers != 0 # go back right
+          $stdout.flush
+          @current_percentage = new_percentage
+        end
+
+        if new_markers > @current_markers
+          print '.'  * (new_markers - @current_markers)
+          @current_markers = new_markers
+          $stdout.flush
+        end
+        if @current_steps == @max_steps
+          print '.' * (max_markers - @current_markers) + ' '
+          $stdout.flush
+        end
+      end
+    end
+  end
+end

data/lib/rubyrep/scan_progress_printers/scan_progress_printers.rb

@@ -0,0 +1,65 @@
+module RR
+  # Manages scan progress printers. Scan progress printers implement functionality
+  # to report the progress of a table scan.
+  # *Each* table scan is handled by a *separate* printer instance.
+  #
+  # Scan progress printers need to register themselves and their command line options
+  # with #register.
+  #
+  # A scan progress printer needs to implement at the minimum the following
+  # functionality:
+  #
+  #   # Receives the command line argument as yielded by OptionParser#on.
+  #   def self.arg=(arg)
+  #
+  #   # Creation of a new ScanProgressPrinter.
+  #   # * +max_steps+: number of steps at completion
+  #   # * +session+: the current Session
+  #   # * +left_table+: name of the left database table
+  #   # * +right_table+: name of the right database table
+  #   def initialize(max_steps, left_table, right_table)
+  #
+  #   # Progress is advanced by +progress+ number of steps.
+  #   def step(progress)
+  #
+  module ScanProgressPrinters
+
+    # Hash of registered ScanProgressPrinters.
+    # Each entry is a hash with the following key and related value:
+    # * key: Identifier of the progress printer
+    # * value: A hash with payload information. Possible values:
+    #   * :+printer_class+: The ScanProgressPrinter class.
+    #   * :+opts+: An array defining the command line options (handed to OptionParter#on).
+    def self.printers
+      @@progress_printers ||= {}
+    end
+
+    # Needs to be called by ScanProgressPrinters to register themselves (+printer+)
+    # and their command line options.
+    # * :+printer_id+ is the symbol through which the printer can be referenced.
+    # * :+printer_class+ is the ScanProgressPrinter class,
+    # * :+opts+ is an array defining the command line options (handed to OptionParter#on).
+    def self.register(printer_id, printer_class, *opts)
+      printers[printer_id] = {
+        :printer_class => printer_class,
+        :opts => opts
+      }
+    end
+
+    # Registers all report printer command line options into the given
+    # OptionParser.
+    # Once the command line is parsed with OptionParser#parse! it will
+    # yield the correct printer class.
+    #
+    # Note:
+    # If multiple printers are specified in the command line, all are yielded.
+    def self.on_printer_selection(opts)
+      printers.each_value do |printer|
+        opts.on(*printer[:opts]) do |arg|
+          printer[:printer_class].arg = arg
+          yield printer[:printer_class]
+        end
+      end
+    end
+  end
+end

data/lib/rubyrep/scan_report_printers/scan_detail_reporter.rb

@@ -0,0 +1,111 @@
+require 'fileutils'
+require 'tmpdir'
+require 'tempfile'
+require 'yaml'
+
+module RR::ScanReportPrinters
+  # A ScanReportPrinter producing a summary (number of differences) only.
+  class ScanDetailReporter < ScanSummaryReporter
+    
+    # Register ScanSummaryReporter with the given command line options.
+    # (Command line format as specified by OptionParser#on.)
+    RR::ScanReportPrinters.register self, "-d", "--detailed[=mode]",
+      "Print the number of differences of each table. E. g.",
+      "  left_table / right_table [differences]",
+      "followed by a full dump of the differences in YAML format.",
+      "The 'mode' argument determines how the row differences are printed:",
+      " * full shows the full records",
+      " * keys shows the primary key columns only",
+      " * diff shows the primary key and differing columsn only"
+    
+    # The current Session object
+    attr_accessor :session
+
+    # The temporary File receiving the differences
+    attr_accessor :tmpfile
+
+    # Mode of reporting. Should be either
+    # * :+full+
+    # * :+keys+ or
+    # * :+diff+
+    attr_accessor :report_mode
+
+    # Array of names of the primary key columns of the table currently being
+    # scanned.
+    attr_accessor :primary_key_names
+
+    # A scan run is to be started using this scan result printer.
+    # +arg+ is the command line argument as yielded by OptionParser#on.
+    def initialize(session, arg)
+      super session, ""
+      self.session = session
+
+      self.report_mode = case arg
+      when 'diff'
+        :diff
+      when 'keys'
+        :keys
+      else
+        :full
+      end
+    end
+    
+    # A scan of the given 'left' table and corresponding 'right' table is executed.
+    # Needs to yield so that the actual scan can be executed.
+    def scan(left_table, right_table)
+
+      super left_table, right_table
+
+    ensure
+      self.primary_key_names = nil
+      if self.tmpfile
+        self.tmpfile.close
+        self.tmpfile.open
+        self.tmpfile.each_line {|line| puts line}
+        self.tmpfile.close!
+        self.tmpfile = nil
+      end
+    end
+
+    # Returns a cleaned row as per current +report_mode+.
+    # +row+ is either a column_name => value hash or an array of 2 such rows.
+    def clear_columns(row)
+      case report_mode
+      when :full
+        row
+      when :keys
+        row = row[0] if row.kind_of?(Array)
+        self.primary_key_names ||= session.left.primary_key_names(self.left_table)
+        row.reject {|column, value| !self.primary_key_names.include?(column)}
+      when :diff
+        self.primary_key_names ||= session.left.primary_key_names(self.left_table)
+        if row.kind_of?(Array)
+          new_row_array = [{}, {}]
+          row[0].each do |column, value|
+            if self.primary_key_names.include?(column) or value != row[1][column]
+              new_row_array[0][column] = row[0][column]
+              new_row_array[1][column] = row[1][column]
+            end
+          end
+          new_row_array
+        else
+          row
+        end
+      end
+    end
+
+    # Each difference is handed to the printer as described in the format
+    # as described e. g. in DirectTableScan#run
+    def report_difference(type, row)
+      self.tmpfile ||= Tempfile.new 'rubyrep_scan_details'
+      tmpfile.puts({type => clear_columns(row)}.to_yaml)
+      super type, row
+    end
+
+    # Optional method. If a scan report printer has it, it is called after the
+    # last table scan is executed.
+    # (A good place to print a final summary.)
+    def scanning_finished
+    end
+  end
+end

data/lib/rubyrep/scan_report_printers/scan_report_printers.rb

@@ -0,0 +1,67 @@
+module RR
+  # Manages scan report printers. Scan report printers implement functionality
+  # to report the row differences identified during a scan.
+  # *All* table scans are processed by the *same* printer instance.
+  #
+  # Scan report printers need to register themselves and their command line options
+  # with #register.
+  #
+  # A scan report printer needs to implement at the minimum the following
+  # functionality:
+  #
+  #   # Creation of a new ScanReportPrinter.
+  #   # * +session+: the current Session object
+  #   # * +arg+: command line argument as yielded by OptionParser#on.
+  #   def initialize(arg)
+  #
+  #   # A scan of the given 'left' table and corresponding 'right' table is executed.
+  #   # Needs to yield so that the actual scan can be executed.
+  #   def scan(left_table, right_table)
+  #
+  #   # Each difference is handed to the printer as described in the format
+  #   # as described e. g. in DirectTableScan#run
+  #   def report_difference(type, row)
+  #
+  #   # Optional method. If a scan report printer has it, it is called after the
+  #   # last table scan is executed.
+  #   # (A good place to print a final summary.)
+  #   def scanning_finished
+  #
+  module ScanReportPrinters
+
+    # Array of registered ScanReportPrinters.
+    # Each entry is a hash with the following keys and related values:
+    # * :+printer_class+: The ScanReportPrinter class.
+    # * :+opts+: An array defining the command line options (handed to OptionParter#on).
+    def self.printers
+      @@report_printers ||= []
+    end
+
+    # Needs to be called by ScanReportPrinters to register themselves (+printer+)
+    # and their command line options.
+    # * :+printer_class+ is the ScanReportPrinter class,
+    # * :+opts+ is an array defining the command line options (handed to OptionParter#on).
+    def self.register(printer_class, *opts)
+      printers << {
+        :printer_class => printer_class,
+        :opts => opts
+      }
+    end
+
+    # Registers all report printer command line options into the given
+    # OptionParser.
+    # Once the command line is parsed with OptionParser#parse! it will
+    # yield the printer class and the optional command line parameter.
+    #
+    # Note:
+    # If multiple printers are specified in the command line, all are created
+    # and yielded.
+    def self.on_printer_selection(opts)
+      printers.each do |printer|
+        opts.on(*printer[:opts]) do |arg|
+          yield printer[:printer_class], arg
+        end
+      end
+    end
+  end
+end

data/lib/rubyrep/scan_report_printers/scan_summary_reporter.rb

@@ -0,0 +1,75 @@
+module RR::ScanReportPrinters
+  # A ScanReportPrinter producing a summary (number of differences) only.
+  class ScanSummaryReporter
+    
+    # Register ScanSummaryReporter with the given command line options.
+    # (Command line format as specified by OptionParser#on.)
+    RR::ScanReportPrinters.register self, "-s", "--summary[=detailed]",
+        "Print the number of differences of each table. Either totals only, e. g.",
+        "  left_table / right_table [differences]",
+        "or a detailed split by type, e. g.",
+        "  left_table / right_table [conflicts] [left_only records] [right_only records]"
+    
+    # Set to true if only the total number of differences should be reported
+    attr_accessor :only_totals
+    
+    # Name of the left table of the current scan
+    attr_accessor :left_table
+    
+    # Name of the right table of the current scan
+    attr_accessor :right_table
+    
+    # Hold the result of the current scan. A hash with a running count of
+    #  +:conflict+, +:left+ (only) or +:right+ (only) records.
+    attr_accessor :scan_result
+
+    # A scan run is to be started using this scan result printer.
+    # +arg+ is the command line argument as yielded by OptionParser#on.
+    def initialize(_, arg)
+      self.only_totals = (arg != 'detailed')
+    end
+    
+    # A scan of the given 'left' table and corresponding 'right' table is executed.
+    # Needs to yield so that the actual scan can be executed.
+    def scan(left_table, right_table)
+      self.left_table = left_table
+      self.right_table = right_table
+      self.scan_result = {:conflict => 0, :left => 0, :right => 0}
+
+      header = left_table.clone
+      header << " / " << right_table if left_table != right_table
+      $stdout.write "#{header.rjust(36)} "
+
+      yield # Give control back so that the actual table scan can be done.
+
+      if only_totals
+        $stdout.write \
+          "#{rjust_value(scan_result[:conflict] + scan_result[:left] + scan_result[:right])}"
+      else
+        $stdout.write \
+          "#{rjust_value(scan_result[:conflict])} " +
+          "#{rjust_value(scan_result[:left])} " +
+          "#{rjust_value(scan_result[:right])}"
+      end
+      $stdout.puts
+    end
+
+    # Right adjusts the given number and returns according string.
+    def rjust_value(value)
+      value.to_s.rjust(3)
+    end
+    private :rjust_value
+
+    # Each difference is handed to the printer as described in the format
+    # as described e. g. in DirectTableScan#run
+    def report_difference(type, row)
+      scan_result[type] += 1
+    end
+
+    # Optional method. If a scan report printer has it, it is called after the
+    # last table scan is executed.
+    # (A good place to print a final summary.)
+    def scanning_finished
+    end
+  end
+end

data/lib/rubyrep/scan_runner.rb

@@ -0,0 +1,25 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+
+module RR
+  # This class implements the functionality of the rrscan.rb command.
+  class ScanRunner < BaseRunner
+
+    CommandRunner.register 'scan' => {
+      :command => self,
+      :description => 'Scans for differing records between databases'
+    }
+
+    # Returns summary description string for the scan command.
+    def summary_description
+      "Scans for differences of the specified tables between both databases."
+    end
+    
+    # Creates the correct scan class.
+    # Parameters as defined under BaseRunner#create_processor
+    def create_processor(left_table, right_table)
+      TableScanHelper.scan_class(session).new session, left_table, right_table
+    end
+  end
+end
+
+

data/lib/rubyrep/session.rb

@@ -0,0 +1,230 @@
+require 'drb'
+
+module RR
+
+  # This class represents a rubyrep session.
+  # Creates and holds expensive objects like e. g. database connections.
+  class Session
+    
+    # The Configuration object provided to the initializer
+    attr_accessor :configuration
+    
+    # Returns the "left" ActiveRecord / proxy database connection
+    def left
+      @connections[:left]
+    end
+    
+    # Stores the "left" ActiveRecord /proxy database connection
+    def left=(connection)
+      @connections[:left] = connection
+    end
+    
+    # Returns the "right" ActiveRecord / proxy database connection
+    def right
+      @connections[:right]
+    end
+    
+    # Stores the "right" ActiveRecord / proxy database connection
+    def right=(connection)
+      @connections[:right] = connection
+    end
+    
+    # Hash to hold under either :left or :right the according Drb / direct DatabaseProxy
+    attr_accessor :proxies
+
+    # Creates a hash of manual primary key names as can be specified with the
+    # Configuration options :+primary_key_names+ or :+auto_key_limit+.
+    # * +db_arm: should be either :left or :right
+    #
+    # Returns the identified manual primary keys. This is a hash with
+    # * key: table_name
+    # * value: array of primary key names
+    def manual_primary_keys(db_arm)
+      manual_primary_keys = {}
+      resolver = TableSpecResolver.new self
+      table_pairs = resolver.resolve configuration.included_table_specs, [], false
+      table_pairs.each do |table_pair|
+        options = configuration.options_for_table(table_pair[:left])
+        key_names = options[:key]
+        if key_names == nil and options[:auto_key_limit] > 0
+          if left.primary_key_names(table_pair[:left], :raw => true).empty?
+            column_names = left.column_names(table_pair[:left])
+            if column_names.size <= options[:auto_key_limit]
+              key_names = column_names
+            end
+          end
+        end
+        if key_names
+          table_name = table_pair[db_arm]
+          manual_primary_keys[table_name] = [key_names].flatten
+        end
+      end
+      manual_primary_keys
+    end
+
+    # Returns the corresponding table in the other database.
+    # * +db_arm+: database of the given table (either :+left+ or :+right+)
+    # * +table+: name of the table
+    #
+    # If no corresponding table can be found, return the given table.
+    # Rationale:
+    # Support the case where a table was dropped from the configuration but
+    # there were still some unreplicated changes left.
+    def corresponding_table(db_arm, table)
+      unless @table_map
+        @table_map = {:left => {}, :right => {}}
+        resolver = TableSpecResolver.new self
+        table_pairs = resolver.resolve configuration.included_table_specs, [], false
+        table_pairs.each do |table_pair|
+          @table_map[:left][table_pair[:left]] = table_pair[:right]
+          @table_map[:right][table_pair[:right]] = table_pair[:left]
+        end
+      end
+      @table_map[db_arm][table] || table
+    end
+    
+    # Returns +true+ if proxy connections are used
+    def proxied?
+      [configuration.left, configuration.right].any? \
+        {|arm_config| arm_config.include? :proxy_host}
+    end
+
+    # Returns an array of table pairs of the configured tables.
+    # Refer to TableSpecResolver#resolve for a detailed description of the
+    # return value.
+    # If +included_table_specs+ is provided (that is: not an empty array), it
+    # will be used instead of the configured table specs.
+    def configured_table_pairs(included_table_specs = [])
+      resolver = TableSpecResolver.new self
+      included_table_specs = configuration.included_table_specs if included_table_specs.empty?
+      resolver.resolve included_table_specs, configuration.excluded_table_specs
+    end
+
+    # Orders the array of table pairs as per primary key / foreign key relations
+    # of the tables. Returns the result.
+    # Only sorts if the configuration has set option :+table_ordering+.
+    # Refer to TableSpecResolver#resolve for a detailed description of the
+    # parameter and return value.
+    def sort_table_pairs(table_pairs)
+      if configuration.options[:table_ordering]
+        left_tables = table_pairs.map {|table_pair| table_pair[:left]}
+        sorted_left_tables = TableSorter.new(self, left_tables).sort
+        sorted_left_tables.map do |left_table|
+          table_pairs.find do |table_pair|
+            table_pair[:left] == left_table
+          end
+        end
+      else
+        table_pairs
+      end
+    end
+
+    # Returns +true+ if the specified database connection is not alive.
+    # * +database+: target database (either +:left+ or :+right+)
+    def database_unreachable?(database)
+      unreachable = true
+      Thread.new do
+        begin
+          if send(database) && send(database).select_one("select 1+1 as x")['x'].to_i == 2
+            unreachable = false # database is actually reachable
+          end
+        end rescue nil
+      end.join configuration.options[:database_connection_timeout]
+      unreachable
+    end
+
+    # Disconnects both database connections
+    def disconnect_databases
+      [:left, :right].each do |database|
+        disconnect_database(database)
+      end
+    end
+
+    # Disconnnects the specified database
+    # * +database+: the target database (either :+left+ or :+right+)
+    def disconnect_database(database)
+      proxy, connection = @proxies[database], @connections[database]
+      @proxies[database] = nil
+      @connections[database] = nil
+      if proxy
+        proxy.destroy_session(connection)
+      end
+    end
+
+    # Refreshes both database connections
+    # * +options+: A options hash with the following settings
+    #   * :+forced+: if +true+, always establish a new database connection
+    def refresh(options = {})
+      [:left, :right].each {|database| refresh_database_connection database, options}
+    end
+
+    # Refreshes the specified database connection.
+    # (I. e. reestablish if not active anymore.)
+    # * +database+: target database (either :+left+ or :+right+)
+    # * +options+: A options hash with the following settings
+    #   * :+forced+: if +true+, always establish a new database connection
+    def refresh_database_connection(database, options)
+      if options[:forced] or database_unreachable?(database)
+        # step 1: disconnect both database connection (if still possible)
+        begin
+          Thread.new do
+            disconnect_database database rescue nil
+          end.join configuration.options[:database_connection_timeout]
+        end
+
+        connect_exception = nil
+        # step 2: try to reconnect the database
+        Thread.new do
+          begin
+            connect_database database
+          rescue Exception => e
+            # save exception so it can be rethrown outside of the thread
+            connect_exception = e
+          end
+        end.join configuration.options[:database_connection_timeout]
+        raise connect_exception if connect_exception
+
+        # step 3: verify if database connections actually work (to detect silent connection failures)
+        if database_unreachable?(database)
+          raise "no connection to '#{database}' database"
+        end
+      end
+    end
+
+    # Set up the (proxied or direct) database connections to the specified
+    # database.
+    # * +database+: the target database (either :+left+ or :+right+)
+    def connect_database(database)
+      if configuration.left == configuration.right and database == :right
+        # If both database configurations point to the same database
+        # then don't create the database connection twice.
+        # Assumes that the left database is always connected before the right one.
+        self.right = self.left
+      else
+        # Connect the database / proxy
+        arm_config = configuration.send database
+        if arm_config.include? :proxy_host
+          drb_url = "druby://#{arm_config[:proxy_host]}:#{arm_config[:proxy_port]}"
+          @proxies[database] = DRbObject.new nil, drb_url
+        else
+          # Create fake proxy
+          @proxies[database] = DatabaseProxy.new
+        end
+        @connections[database] = @proxies[database].create_session arm_config
+
+        send(database).manual_primary_keys = manual_primary_keys(database)
+      end
+    end
+        
+    # Creates a new rubyrep session with the provided Configuration
+    def initialize(config = Initializer::configuration)
+      @connections = {:left => nil, :right => nil}
+      @proxies = {:left => nil, :right => nil}
+      
+      # Keep the database configuration for future reference
+      self.configuration = config
+
+      refresh
+    end
+  end
+end