rubyrep 1.0.0

data/lib/rubyrep/proxy_connection.rb

@@ -0,0 +1,318 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
+
+require 'drb'
+
+require 'rubyrep'
+require 'forwardable'
+
+require 'active_record/connection_adapters/abstract_adapter'
+
+module RR
+
+  # This class represents a remote activerecord database connection.
+  # Normally created by DatabaseProxy
+  class ProxyConnection
+    extend Forwardable
+    
+    # The database connection
+    attr_accessor :connection
+
+    # A hash as described by ActiveRecord::Base#establish_connection
+    attr_accessor :config
+    
+    # Forward certain methods to the proxied database connection
+    def_delegators :connection,
+      :columns, :quote_column_name,
+      :quote_table_name, :execute,
+      :select_one, :select_all, :tables,
+      :begin_db_transaction, :rollback_db_transaction, :commit_db_transaction,
+      :referenced_tables,
+      :create_or_replace_replication_trigger_function,
+      :create_replication_trigger, :drop_replication_trigger, :replication_trigger_exists?,
+      :sequence_values, :update_sequences, :clear_sequence_setup,
+      :create_table, :drop_table, :add_big_primary_key
+    
+    # Caching the primary keys. This is a hash with
+    #   * key: table name
+    #   * value: array of primary key names
+    attr_accessor :primary_key_names_cache
+    
+    # Hash to register cursors.
+    # Purpose:
+    #   Objects only referenced remotely via DRb can be garbage collected.
+    #   We register them in this hash to protect them from unintended garbage collection.
+    attr_accessor :cursors
+    
+    # 2-level Hash of table_name => column_name => Column objects.
+    attr_accessor :table_columns
+    
+    # Hash of table_name => array of column names pairs.
+    attr_accessor :table_column_names
+
+    # A hash of manually overwritten primary keys:
+    # * key: table_name
+    # * value: array of primary key names
+    attr_accessor :manual_primary_keys
+    
+    # Returns an array of primary key names for the given +table_name+.
+    # Caches the result for future calls. Allows manual overwrites through
+    # the Configuration options +:primary_key_names+ or :+primary_key_only_limit+.
+    #
+    # Parameters:
+    # * +table_name+: name of the table
+    # * +options+: An option hash with the following valid options:
+    #   * :+raw+: if +true+, than don't use manual overwrites and don't cache
+    def primary_key_names(table_name, options = {})
+      return connection.primary_key_names(table_name) if options[:raw]
+      
+      self.primary_key_names_cache ||= {}
+      result = primary_key_names_cache[table_name]
+      unless result
+        result = manual_primary_keys[table_name] || connection.primary_key_names(table_name)
+        primary_key_names_cache[table_name] = result
+      end
+      result
+    end
+    
+    # Returns a Hash of currently registerred cursors
+    def cursors
+      @cursors ||= {}
+    end
+    
+    # Store a cursor in the register to protect it from the garbage collector.
+    def save_cursor(cursor)
+      cursors[cursor] = cursor
+    end
+
+    # default number of records that are read into memory at a time during
+    # select queries
+    DEFAULT_ROW_BUFFER_SIZE = 1000
+
+    # Returns a cusor as produced by the #select_cursor method of the connection
+    # extenders.
+    #
+    # Two modes of operation: Either
+    # * execute the specified query (takes precedense) OR
+    # * first build the query based on options forwarded to #table_select_query
+    # +options+ is a hash with
+    # * :+query+: executes the given query
+    # * :+type_cast+: if +true+, build a type casting cursor around the result
+    # * :+table+: name of the table from which to read data
+    # * further options as taken by #table_select_query to build the query
+    # * :+row_buffer_size+:
+    #   Integer controlling how many rows a read into memory at one time.
+    def select_cursor(options)
+      row_buffer_size = options[:row_buffer_size] || DEFAULT_ROW_BUFFER_SIZE
+      query = options[:query] || table_select_query(options[:table], options)
+      cursor = connection.select_cursor query, row_buffer_size
+      if options[:type_cast]
+        cursor = TypeCastingCursor.new(self, options[:table], cursor)
+      end
+      cursor
+    end
+    
+    # Create a session on the proxy side according to provided configuration hash.
+    # +config+ is a hash as described by ActiveRecord::Base#establish_connection
+    def initialize(config)
+      self.connection = ConnectionExtenders.db_connect config
+      self.config = config
+      self.manual_primary_keys = {}
+    end
+
+    # Checks if the connection is still active and if not, reestablished it.
+    def refresh
+      unless self.connection.active?
+        self.connection = ConnectionExtenders.db_connect config
+      end
+    end
+    
+    # Destroys the session
+    def destroy
+      self.connection.disconnect!
+      
+      cursors.each_key do |cursor|
+        cursor.destroy
+      end
+      cursors.clear
+    end
+    
+    # Quotes the given value. It is assumed that the value belongs to the specified column name and table name.
+    # Caches the column objects for higher speed.
+    def quote_value(table, column, value)
+      self.table_columns ||= {}
+      unless table_columns.include? table
+        table_columns[table] = {}
+        columns(table).each {|c| table_columns[table][c.name] = c}
+      end
+      connection.quote value, table_columns[table][column]
+    end
+    
+    # Create a cursor for the given table.
+    #   * +cursor_class+: should specify the Cursor class (e. g. ProxyBlockCursor or ProxyRowCursor).
+    #   * +table+: name of the table 
+    #   * +options+: An option hash that is used to construct the SQL query. See ProxyCursor#construct_query for details.
+    def create_cursor(cursor_class, table, options = {})
+      cursor = cursor_class.new self, table
+      cursor.prepare_fetch options
+      save_cursor cursor
+      cursor
+    end
+    
+    # Destroys the provided cursor and removes it from the register
+    def destroy_cursor(cursor)
+      cursor.destroy
+      cursors.delete cursor
+    end
+    
+    # Returns an array of column names of the given table name.
+    # The array is ordered in the sequence as returned by the database.
+    # The result is cached for higher speed.
+    def column_names(table)
+      self.table_column_names ||= {}
+      unless table_column_names.include? table
+        table_column_names[table] = columns(table).map {|c| c.name}
+      end
+      table_column_names[table]
+    end
+  
+    # Returns a list of quoted column names for the given +table+ as comma 
+    # separated string.
+    def quote_column_list(table)
+      column_names(table).map do |column_name| 
+        quote_column_name(column_name)
+      end.join(', ')
+    end
+    private :quote_column_list
+    
+    # Returns a list of quoted primary key names for the given +table+ as comma
+    # separated string.
+    def quote_key_list(table)
+      primary_key_names(table).map do |column_name| 
+        quote_column_name(column_name)
+      end.join(', ')
+    end
+    private :quote_key_list
+    
+    
+    # Generates an sql condition string for the given +table+ based on
+    #   * +row+: a hash of primary key => value pairs designating the target row
+    #   * +condition+: the type of sql condition (something like '>=' or '=', etc.)
+    def row_condition(table, row, condition)
+      query_part = ""
+      query_part << ' (' << quote_key_list(table) << ') ' << condition
+      query_part << ' (' << primary_key_names(table).map do |key|
+        quote_value(table, key, row[key])
+      end.join(', ') << ')'
+      query_part
+    end
+    private :row_condition
+
+    # Returns an SQL query string for the given +table+ based on the provided +options+.
+    # +options+ is a hash that can contain any of the following:
+    #   * :+from+: nil OR the hash of primary key => value pairs designating the start of the selection
+    #   * :+exclude_starting_row+: if true, do not include the row specified by :+from+
+    #   * :+to+: nil OR the hash of primary key => value pairs designating the end of the selection
+    #   * :+row_keys+: an array of primary key => value hashes specify the target rows.
+    def table_select_query(table, options = {})
+      query = "select #{quote_column_list(table)}"
+      query << " from #{quote_table_name(table)}"
+      query << " where" if [:from, :to, :row_keys].any? {|key| options.include? key}
+      first_condition = true
+      if options[:from]
+        first_condition = false
+        matching_condition = options[:exclude_starting_row] ? '>' : '>='
+        query << row_condition(table, options[:from], matching_condition)
+      end
+      if options[:to]
+        query << ' and' unless first_condition
+        first_condition = false
+        query << row_condition(table, options[:to], '<=')
+      end
+      if options[:row_keys]
+        query << ' and' unless first_condition
+        if options[:row_keys].empty?
+          query << ' false'
+        else
+          query << ' (' << quote_key_list(table) << ') in ('
+          first_key = true
+          options[:row_keys].each do |row|
+            query << ', ' unless first_key
+            first_key = false
+            query << '(' << primary_key_names(table).map do |key|
+              quote_value(table, key, row[key])
+            end.join(', ') << ')'
+          end
+          query << ')'
+        end
+      end
+      query << " order by #{quote_key_list(table)}"
+
+      query
+    end
+    
+    # Returns an SQL insert query for the given +table+ and +values+.
+    # +values+ is a hash of column_name => value pairs.
+    def table_insert_query(table, values)
+      query = "insert into #{quote_table_name(table)}"
+      query << '(' << values.keys.map do |column_name|
+        quote_column_name(column_name)
+      end.join(', ') << ') '
+      query << 'values(' << values.map do |column_name, value|
+        quote_value(table, column_name, value)
+      end.join(', ') << ')'
+      query
+    end
+    
+    # Inserts the specified records into the named +table+.
+    # +values+ is a hash of column_name => value pairs.
+    def insert_record(table, values)
+      execute table_insert_query(table, values)
+    end
+    
+    # Returns an SQL update query.
+    # * +table+: name of the target table
+    # * +values+: a hash of column_name => value pairs
+    # * +org_key+:
+    #   A hash of column_name => value pairs. If +nil+, use the key specified by
+    #   +values+ instead.
+    def table_update_query(table, values, org_key = nil)
+      org_key ||= values
+      query = "update #{quote_table_name(table)} set "
+      query << values.map do |column_name, value|
+        "#{quote_column_name(column_name)} = #{quote_value(table, column_name, value)}"
+      end.join(', ')
+      query << " where (" << quote_key_list(table) << ") = ("
+      query << primary_key_names(table).map do |key|
+        quote_value(table, key, org_key[key])
+      end.join(', ') << ")"
+    end
+    
+    # Updates the specified records of the specified table.
+    # * +table+: name of the target table
+    # * +values+: a hash of column_name => value pairs.
+    # * +org_key+:
+    #   A hash of column_name => value pairs. If +nil+, use the key specified by
+    #   +values+ instead.
+    def update_record(table, values, org_key = nil)
+      execute table_update_query(table, values, org_key)
+    end
+
+    # Returns an SQL delete query for the given +table+ and +values+
+    # +values+ is a hash of column_name => value pairs. (Only the primary key
+    # values will be used and must be included in the hash.)
+    def table_delete_query(table, values)
+      query = "delete from #{quote_table_name(table)}"
+      query << " where (" << quote_key_list(table) << ") = ("
+      query << primary_key_names(table).map do |key|
+        quote_value(table, key, values[key])
+      end.join(', ') << ")"
+    end
+    
+    # Deletes the specified record from the named +table+.
+    # +values+ is a hash of column_name => value pairs. (Only the primary key
+    # values will be used and must be included in the hash.)
+    def delete_record(table, values)
+      execute table_delete_query(table, values)
+    end
+  end
+end

data/lib/rubyrep/proxy_cursor.rb

@@ -0,0 +1,44 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
+
+require 'rubyrep'
+
+module RR
+  # Provides shared functionality for ProxyRowCursor and ProxyBlockCursor
+  class ProxyCursor
+    
+    # The current ProxyConnection.
+    attr_accessor :connection
+    
+    # The name of the current table.
+    attr_accessor :table
+    
+    # Array of primary key names for current table.
+    attr_accessor :primary_key_names
+    
+    # The current cursor.
+    attr_accessor :cursor
+    
+    # Shared initializations 
+    #   * connection: the current proxy connection
+    #   * table: table_name
+    def initialize(connection, table)
+      self.connection = connection
+      self.table = table
+      self.primary_key_names = connection.primary_key_names table
+    end
+    
+    # Initiate a query for the specified row range.
+    # +options+: An option hash that is used to construct the SQL query. See ProxyCursor#construct_query for details.
+    def prepare_fetch(options = {})
+      self.cursor = connection.select_cursor(
+        options.merge(:table => table, :type_cast => true)
+      )
+    end
+    
+    # Releases all ressources
+    def destroy
+      self.cursor.clear if self.cursor
+      self.cursor = nil
+    end
+  end
+end

data/lib/rubyrep/proxy_row_cursor.rb

@@ -0,0 +1,43 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
+
+require 'digest/sha1'
+
+require 'rubyrep'
+
+module RR
+  
+  # This class is used to scan a given table range 
+  # Can return rows either themselves or only their checksum
+  class ProxyRowCursor < ProxyCursor
+    
+    # The column_name => value hash of the current row.
+    attr_accessor :current_row
+    
+    # Creates a new cursor
+    #   * session: the current proxy session
+    #   * table: table_name
+    def initialize(session, table)
+      super
+    end
+    
+    # Returns true if there are unprocessed rows in the table range
+    def next?
+      cursor.next?
+    end
+    
+    # Returns the next row in cursor
+    def next_row
+      cursor.next_row
+    end
+    
+    # Returns for the next row
+    #   * a hash of :column_name => value pairs of the primary keys
+    #   * checksum string for that row
+    def next_row_keys_and_checksum
+      self.current_row = cursor.next_row
+      keys = self.current_row.reject {|key, | not primary_key_names.include? key}
+      checksum = Digest::SHA1.hexdigest(Marshal.dump(self.current_row))
+      return keys, checksum
+    end
+  end
+end

data/lib/rubyrep/proxy_runner.rb

@@ -0,0 +1,89 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'optparse'
+require 'drb'
+
+module RR
+  # This class implements the functionality of the rrproxy.rb command.
+  class ProxyRunner
+
+    CommandRunner.register 'proxy' => {
+      :command => self,
+      :description => 'Proxies connections from rubyrep commands to the database'
+    }
+    
+    # Default options to start a DatabaseProxy server
+    DEFAULT_OPTIONS = {
+      :port => DatabaseProxy::DEFAULT_PORT,
+      :host => ''
+    }
+    
+    # Parses the given command line parameter array.
+    # Returns 
+    #   * the options hash or nil if command line parsing failed
+    #   * status (as per UNIX conventions: 1 if parameters were invalid, 0 otherwise)
+    def get_options(args)
+      options = DEFAULT_OPTIONS
+      status = 0
+
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: #{$0} proxy [options]"
+        opts.separator ""
+        opts.separator "Specific options:"
+
+        opts.on("-h","--host", "=IP_ADDRESS", "IP address to listen on. Default: binds to all IP addresses of the computer") do |arg|
+          options[:host] = arg
+        end
+
+        opts.on("-p","--port", "=PORT_NUMBER", Integer, "TCP port to listen on. Default port: #{DatabaseProxy::DEFAULT_PORT}") do |arg|
+          options[:port] = arg
+        end
+
+        opts.on_tail("--help", "Show this message") do
+          $stderr.puts opts
+          options = nil
+        end
+      end
+
+      begin
+        parser.parse!(args)
+      rescue Exception => e
+        $stderr.puts "Command line parsing failed: #{e}"
+        $stderr.puts parser.help
+        options = nil
+        status = 1
+      end
+  
+      return options, status
+    end
+
+    # Builds the druby URL from the given options and returns it
+    def build_url(options)
+      "druby://#{options[:host]}:#{options[:port]}"
+    end
+
+    # Starts a proxy server under the given druby URL
+    def start_server(url)
+      proxy = DatabaseProxy.new
+
+      DRb.start_service(url, proxy)
+      DRb.thread.join
+    end
+    
+    # Runs the ProxyRunner (processing of command line & starting of server)
+    # args: the array of command line options with which to start the server
+    def self.run(args)
+      runner = ProxyRunner.new
+      
+      options, status = runner.get_options(args)
+      if options
+        url = runner.build_url(options)
+        runner.start_server(url)
+      end
+      status
+    end
+
+  end
+end
+
+

data/lib/rubyrep/replication_difference.rb

@@ -0,0 +1,91 @@
+module RR
+
+  # Describes a (record specific) difference between both databases as identifed
+  # via change log.
+  class ReplicationDifference
+
+    # The current Session.
+    attr_accessor :session
+    
+    # The type of the difference. Either
+    # * :+left+: change in left database
+    # * :+right+: change in right database
+    # * :+conflict+: change in both databases
+    # * :+no_diff+: changes in both databases constitute no difference
+    attr_accessor :type
+
+    # A hash with keys :+left+ and / or :+right+.
+    # Hash values are LoggedChange instances.
+    def changes
+      @changes ||= {}
+    end
+
+    # Creates a new ReplicationDifference instance.
+    # +session+ is the current Session.
+    def initialize(session)
+      self.session = session
+    end
+
+    # Should be set to +true+ if this ReplicationDifference instance was
+    # successfully loaded.
+    attr_writer :loaded
+
+    # Returns +true+ if a replication difference was loaded
+    def loaded?
+      @loaded
+    end
+
+    # Shortcut to calculate the "other" database.
+    OTHER_SIDE = {
+      :left => :right,
+      :right => :left
+    }
+
+    # Resulting diff type based on types of left changes (outer hash) and right
+    # changes (inner hash)
+    DIFF_TYPES = {
+      :insert =>    {:insert => :conflict, :update => :conflict, :delete => :conflict,  :no_change => :left},
+      :update =>    {:insert => :conflict, :update => :conflict, :delete => :conflict,  :no_change => :left},
+      :delete =>    {:insert => :conflict, :update => :conflict, :delete => :no_change, :no_change => :left},
+      :no_change => {:insert => :right,    :update => :right,    :delete => :right,     :no_change => :no_change}
+    }
+
+    # Amends a difference according to new entries in the change log table
+    def amend
+      session.reload_changes
+      changes[:left].load
+      changes[:right].load
+      self.type = DIFF_TYPES[changes[:left].type][changes[:right].type]
+    end
+    
+    # Loads a difference
+    def load
+      change_times = {}
+      [:left, :right].each do |database|
+        changes[database] = LoggedChange.new session, database
+        change_times[database] = changes[database].oldest_change_time
+      end
+      return if change_times[:left] == nil and change_times[:right] == nil
+
+      oldest = nil
+      [:left, :right].each do |database|
+        oldest = OTHER_SIDE[database] if change_times[database] == nil
+      end
+      oldest ||= change_times[:left] <= change_times[:right] ? :left : :right
+      changes[oldest].load_oldest
+
+      changes[OTHER_SIDE[oldest]].load_specified(
+        session.corresponding_table(oldest, changes[oldest].table),
+        changes[oldest].key)
+
+      self.type = DIFF_TYPES[changes[:left].type][changes[:right].type]
+      self.loaded = true
+    end
+
+    # Prevents session from going into YAML output
+    def to_yaml_properties
+      instance_variables.sort.reject {|var_name| var_name == '@session'}
+    end
+
+  end
+end