rubyrep 1.0.0

data/lib/rubyrep/connection_extenders/postgresql_extender.rb

@@ -0,0 +1,261 @@
+require 'time'
+
+# A cursor to iterate over the records returned by select_cursor.
+# Only one row is kept in memory at a time.
+class PGresult
+  # Returns true if there are more rows to read.
+  def next?
+    @current_row_num ||= 0
+    @num_rows ||= self.ntuples()
+    @current_row_num < @num_rows
+  end
+  
+  # Returns the row as a column => value hash and moves the cursor to the next row.
+  def next_row
+    raise("no more rows available") unless next?
+    row = {}
+    @fields ||= self.fields
+    @fields.each_with_index do |field, field_index| 
+      if self.getisnull(@current_row_num, field_index)
+        value = nil
+      else
+        value = self.getvalue @current_row_num, field_index
+      end
+      
+      row[@fields[field_index]] = value
+    end
+    @current_row_num += 1
+    row
+  end
+end
+
+# Hack:
+# For some reasons these methods were removed in Rails 2.2.2, thus breaking
+# the binary and multi-lingual data loading.
+# So here they are again.
+module ActiveRecord
+  module ConnectionAdapters
+    # PostgreSQL-specific extensions to column definitions in a table.
+    class PostgreSQLColumn < Column #:nodoc:
+
+      # Escapes binary strings for bytea input to the database.
+      def self.string_to_binary(value)
+        if PGconn.respond_to?(:escape_bytea)
+          self.class.module_eval do
+            define_method(:string_to_binary) do |value|
+              PGconn.escape_bytea(value) if value
+            end
+          end
+        else
+          self.class.module_eval do
+            define_method(:string_to_binary) do |value|
+              if value
+                result = ''
+                value.each_byte { |c| result << sprintf('\\\\%03o', c) }
+                result
+              end
+            end
+          end
+        end
+        self.class.string_to_binary(value)
+      end
+
+      # Unescapes bytea output from a database to the binary string it represents.
+      def self.binary_to_string(value)
+        # In each case, check if the value actually is escaped PostgreSQL bytea output
+        # or an unescaped Active Record attribute that was just written.
+        if PGconn.respond_to?(:unescape_bytea)
+          self.class.module_eval do
+            define_method(:binary_to_string) do |value|
+              if value =~ /\\\d{3}/
+                PGconn.unescape_bytea(value)
+              else
+                value
+              end
+            end
+          end
+        else
+          self.class.module_eval do
+            define_method(:binary_to_string) do |value|
+              if value =~ /\\\d{3}/
+                result = ''
+                i, max = 0, value.size
+                while i < max
+                  char = value[i]
+                  if char == ?\\
+                    if value[i+1] == ?\\
+                      char = ?\\
+                      i += 1
+                    else
+                      char = value[i+1..i+3].oct
+                      i += 3
+                    end
+                  end
+                  result << char
+                  i += 1
+                end
+                result
+              else
+                value
+              end
+            end
+          end
+        end
+        self.class.binary_to_string(value)
+      end
+    end
+  end
+end
+
+module RR
+  module ConnectionExtenders
+
+    # Fetches results from a PostgreSQL cursor object.
+    class PostgreSQLFetcher
+
+      # The current database connection
+      attr_accessor :connection
+
+      # Name of the cursor from which to fetch
+      attr_accessor :cursor_name
+
+      # Number of rows to be read at once
+      attr_accessor :row_buffer_size
+
+      # Creates a new fetcher.
+      # * +connection+: the current database connection
+      # * +cursor_name+: name of the cursor from which to fetch
+      # * +row_buffer_size+: number of records to read at once
+      def initialize(connection, cursor_name, row_buffer_size)
+        self.connection = connection
+        self.cursor_name = cursor_name
+        self.row_buffer_size = row_buffer_size
+      end
+
+      # Executes the specified SQL staements, returning the result
+      def execute(sql)
+        connection.execute sql
+      end
+
+      # Returns true if there are more rows to read.
+      def next?
+        @current_result ||= execute("FETCH FORWARD #{row_buffer_size} FROM #{cursor_name}")
+        @current_result.next?
+      end
+
+      # Returns the row as a column => value hash and moves the cursor to the next row.
+      def next_row
+        raise("no more rows available") unless next?
+        row = @current_result.next_row
+        unless @current_result.next?
+          @current_result.clear
+          @current_result = nil
+        end
+        row
+      end
+
+      # Closes the cursor and frees up all ressources
+      def clear
+        if @current_result
+          @current_result.clear
+          @current_result = nil
+        end
+        result = execute("CLOSE #{cursor_name}")
+        result.clear if result
+      end
+    end
+
+    # Provides various PostgreSQL specific functionality required by Rubyrep.
+    module PostgreSQLExtender
+      RR::ConnectionExtenders.register :postgresql => self
+
+      # Executes the given sql query with the otional name written in the 
+      # ActiveRecord log file.
+      #
+      # :+row_buffer_size+ controls how many records are ready into memory at a
+      # time. Implemented using the PostgeSQL "DECLARE CURSOR" and "FETCH" constructs.
+      # This is necessary as the postgresql driver always reads the
+      # complete resultset into memory.
+      #
+      # Returns the results as a Cursor object supporting
+      #   * next? - returns true if there are more rows to read
+      #   * next_row - returns the row as a column => value hash and moves the cursor to the next row
+      #   * clear - clearing the cursor (making allocated memory available for GC)
+      def select_cursor(sql, row_buffer_size = 1000)
+        cursor_name = "RR_#{Time.now.to_i}#{rand(1_000_000)}"
+        execute("DECLARE #{cursor_name} NO SCROLL CURSOR WITH HOLD FOR " + sql)
+        PostgreSQLFetcher.new(self, cursor_name, row_buffer_size)
+      end
+      
+      # Returns an ordered list of primary key column names of the given table
+      def primary_key_names(table)
+        row = self.select_one(<<-end_sql)
+          SELECT relname
+          FROM pg_class
+          WHERE relname = '#{table}'
+        end_sql
+        if row.nil?
+        raise "table '#{table}' does not exist"
+      end
+        
+      row = self.select_one(<<-end_sql)
+          SELECT cons.conkey 
+          FROM pg_class           rel
+          JOIN pg_constraint      cons ON (rel.oid = cons.conrelid)
+          WHERE cons.contype = 'p' AND rel.relname = '#{table}'          
+      end_sql
+      if row.nil?
+      return []
+    end
+    column_parray = row['conkey']
+        
+    # Change a Postgres Array of attribute numbers
+    # (returned in String form, e. g.: "{1,2}") into an array of Integers
+    column_ids = column_parray.sub(/^\{(.*)\}$/,'\1').split(',').map {|a| a.to_i}
+
+    columns = {}
+    rows = self.select_all(<<-end_sql)
+          SELECT attnum, attname
+          FROM pg_class           rel
+          JOIN pg_constraint      cons ON (rel.oid = cons.conrelid)
+          JOIN pg_attribute       attr ON (rel.oid = attr.attrelid and attr.attnum = any (cons.conkey))
+          WHERE cons.contype = 'p' AND rel.relname = '#{table}'
+    end_sql
+    sorted_columns = []
+    if not rows.nil?
+      rows.each() {|r| columns[r['attnum'].to_i] = r['attname']}
+      sorted_columns = column_ids.map {|column_id| columns[column_id]}
+    end
+    sorted_columns
+  end
+
+  # Returns for each given table, which other tables it references via
+  # foreign key constraints.
+  # * tables: an array of table names
+  # Returns: a hash with
+  # * key: name of the referencing table
+  # * value: an array of names of referenced tables
+  def referenced_tables(tables)
+    rows = self.select_all(<<-end_sql)
+          select distinct referencing.relname as referencing_table, referenced.relname as referenced_table
+          from pg_class referencing
+          left join pg_constraint on referencing.oid = pg_constraint.conrelid
+          left join pg_class referenced on pg_constraint.confrelid = referenced.oid
+          where referencing.relkind='r'
+          and referencing.relname in ('#{tables.join("', '")}')
+    end_sql
+    result = {}
+    rows.each do |row|
+      unless result.include? row['referencing_table']
+        result[row['referencing_table']] = []
+      end
+      if row['referenced_table'] != nil
+        result[row['referencing_table']] << row['referenced_table']
+      end
+    end
+    result
+  end
+end
+end
+end
+

data/lib/rubyrep/database_proxy.rb

@@ -0,0 +1,52 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
+
+require 'drb'
+
+require 'rubyrep'
+
+module RR
+  # The proxy to a remote database connection
+  class DatabaseProxy
+    
+    # Ensure that the proxy object always stays on server side and only remote
+    # references are returned to the client.
+    include DRbUndumped 
+    
+    # Default tcp port to listen on
+    DEFAULT_PORT = 9876
+    
+    # A simple Hash to hold Session object
+    # Purpose: preventing them from being garbage collected when they are only referenced through Drb
+    attr_accessor :session_register
+    
+    def initialize
+      self.session_register = {}
+    end
+    
+    # Create a ProxyConnection according to provided configuration Hash.
+    # +config+ is a hash as described by ActiveRecord::Base#establish_connection
+    def create_session(config)
+      session = ProxyConnection.new config
+      self.session_register[session] = session
+      session
+    end
+    
+    # Destroys the given session from the session register
+    def destroy_session(session)
+      session.destroy
+      session_register.delete session
+    end
+    
+    # Returns 'pong'. Used to verify that a working proxy is running.
+    def ping
+      'pong'
+    end
+    
+    # Terminates this proxy
+    def terminate!
+      # AL: The only way I could find to kill the main thread from a sub thread
+      Thread.main.raise SystemExit
+    end
+  end
+end
+

data/lib/rubyrep/direct_table_scan.rb

@@ -0,0 +1,75 @@
+module RR
+
+  # Scans two tables for differences.
+  # Doesn't have any reporting functionality by itself. 
+  # Instead DirectTableScan#run yields all the differences for the caller to do with as it pleases.
+  # Usage:
+  #   1. Create a new DirectTableScan object and hand it all necessary information
+  #   2. Call DirectTableScan#run to do the actual comparison
+  #   3. The block handed to DirectTableScan#run receives all differences
+  class DirectTableScan < TableScan
+    include TableScanHelper
+
+    # The TypeCastingCursor for the left table
+    attr_accessor :left_caster
+    
+    # The TypeCastingCursor for the right table
+    attr_accessor :right_caster
+
+    # Creates a new DirectTableScan instance
+    #   * session: a Session object representing the current database session
+    #   * left_table: name of the table in the left database
+    #   * right_table: name of the table in the right database. If not given, same like left_table
+    def initialize(session, left_table, right_table = nil)
+      super
+    end
+    
+    # Runs the table scan.
+    # Calls the block for every found difference.
+    # Differences are yielded with 2 parameters
+    #   * type: describes the difference, either :left (row only in left table), :right (row only in right table) or :conflict
+    #   * row: For :left or :right cases a hash describing the row; for :conflict an array of left and right row.
+    #          A row is a hash of column_name => value pairs.
+    def run(&blck)
+      left_cursor = right_cursor = nil
+      left_cursor = session.left.select_cursor(
+        :table => left_table,
+        :row_buffer_size => scan_options[:row_buffer_size],
+        :type_cast => true
+      )
+      right_cursor = session.right.select_cursor(
+        :table => right_table,
+        :row_buffer_size => scan_options[:row_buffer_size],
+        :type_cast => true
+      )
+      left_row = right_row = nil
+      update_progress 0 # ensures progress bar is printed even if there are no records
+      while left_row or right_row or left_cursor.next? or right_cursor.next?
+        # if there is no current left row, _try_ to load the next one
+        left_row ||= left_cursor.next_row if left_cursor.next?
+        # if there is no current right row, _try_ to load the next one
+        right_row ||= right_cursor.next_row if right_cursor.next?
+        rank = rank_rows left_row, right_row
+        case rank
+        when -1
+          yield :left, left_row
+          left_row = nil
+          update_progress 1
+        when 1
+          yield :right, right_row
+          right_row = nil
+          update_progress 1
+        when 0
+          update_progress 2
+          if not left_row == right_row
+            yield :conflict, [left_row, right_row]
+          end
+          left_row = right_row = nil
+        end
+        # check for corresponding right rows
+      end
+    ensure
+      [left_cursor, right_cursor].each {|cursor| cursor.clear if cursor}
+    end
+  end
+end

data/lib/rubyrep/generate_runner.rb

@@ -0,0 +1,105 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'optparse'
+
+module RR
+  # This class implements the functionality of the 'generate' command.
+  class GenerateRunner
+
+    CONFIG_TEMPLATE = <<EOF
+RR::Initializer::run do |config|
+  config.left = {
+    :adapter  => 'postgresql', # or 'mysql'
+    :database => 'SCOTT',
+    :username => 'scott',
+    :password => 'tiger',
+    :host     => '172.16.1.1'
+  }
+
+  config.right = {
+    :adapter  => 'postgresql',
+    :database => 'SCOTT',
+    :username => 'scott',
+    :password => 'tiger',
+    :host     => '172.16.1.2'
+  }
+
+  config.include_tables 'dept'
+  config.include_tables /^e/ # regexp matching all tables starting with e
+  # config.include_tables /./ # regexp matching all tables in the database
+end
+EOF
+
+    CommandRunner.register 'generate' => {
+      :command => self,
+      :description => 'Generates a configuration file template'
+    }
+    
+    # Provided options. Possible values:
+    # * +:config_file+: path to config file
+    attr_accessor :options
+
+    # Parses the given command line parameter array.
+    # Returns the status (as per UNIX conventions: 1 if parameters were invalid,
+    # 0 otherwise)
+    def process_options(args)
+      status = 0
+      self.options = {}
+
+      parser = OptionParser.new do |opts|
+        opts.banner = <<EOS
+Usage: #{$0} generate [file_name]
+
+  Generates a configuration file template under name [file_name].
+EOS
+        opts.separator ""
+        opts.separator "  Specific options:"
+
+        opts.on_tail("--help", "Show this message") do
+          $stderr.puts opts
+          self.options = nil
+        end
+      end
+
+      begin
+        unprocessed_args = parser.parse!(args)
+        if options # this will be +nil+ if the --help option is specified
+          raise("Please specify the name of the configuration file") if unprocessed_args.empty?
+          options[:file_name] = unprocessed_args[0]
+        end
+      rescue Exception => e
+        $stderr.puts "Command line parsing failed: #{e}"
+        $stderr.puts parser.help
+        self.options = nil
+        status = 1
+      end
+
+      return status
+    end
+
+    # Generates a configuration file template.
+    def execute
+      if File.exists?(options[:file_name])
+        raise("Cowardly refuse to overwrite existing file '#{options[:file_name]}'")
+      end
+      File.open(options[:file_name], 'w') do |f|
+        f.write CONFIG_TEMPLATE
+      end
+    end
+
+    # Entry points for executing a processing run.
+    # args: the array of command line options that were provided by the user.
+    def self.run(args)
+      runner = new
+
+      status = runner.process_options(args)
+      if runner.options
+        runner.execute
+      end
+      status
+    end
+
+  end
+end
+
+

data/lib/rubyrep/initializer.rb

@@ -0,0 +1,39 @@
+module RR
+
+  # The settings of the current deployment are passed to Rubyrep through the
+  # Initializer::run method.
+  # This method yields a Configuration object for overwriting of the default
+  # settings.
+  # Accordingly a configuration file should look something like this:
+  #
+  #   Rubyrep::Initializer.run do |config|
+  #     config.left = ...
+  #   end
+  class Initializer
+
+    # Sets a new Configuration object
+    # Current configuration values are lost and replaced with the default
+    # settings.
+    def self.reset
+      @@configuration = Configuration.new
+    end
+    reset
+
+    # Returns the current Configuration object
+    def self.configuration
+      @@configuration
+    end
+
+    # Allows direct overwriting of the Configuration
+    def self.configuration=(configuration)
+      @@configuration = configuration
+    end
+
+    # Yields the current Configuration object to enable overwriting of
+    # configuration values.
+    # Refer to the Initializer class documentation for a usage example.
+    def self.run
+      yield configuration
+    end
+  end
+end