andyjeffries-rubyrep 1.2.1

data/lib/rubyrep/logged_change_loader.rb

@@ -0,0 +1,197 @@
+module RR
+  
+  # Makes management of logged change loaders easier
+  class LoggedChangeLoaders
+
+    # The current Session
+    attr_accessor :session
+
+    # A hash of LoggedChangeLoader instances for the :+left+ and :+right+ database
+    attr_accessor :loaders
+
+    # Create new logged change loaders.
+    # * +session+: Current Session
+    def initialize(session)
+      self.session = session
+      self.loaders = {}
+      [:left, :right].each do |database|
+        loaders[database] = LoggedChangeLoader.new(session, database)
+      end
+    end
+
+    # Returns the LoggedChangeLoader for the specified (:+left+ or :+right+)
+    # database.
+    def [](database)
+      loaders[database]
+    end
+
+    # Forces an update of the change log cache
+    def update
+      [:left, :right].each {|database| self[database].update :forced => true}
+    end
+  end
+
+  # Caches the entries in the change log table
+  class LoggedChangeLoader
+
+    # The current +Session+.
+    attr_accessor :session
+
+    # Current database (either :+left+ or :+right+)
+    attr_accessor :database
+
+    # The current +ProxyConnection+.
+    attr_accessor :connection
+
+    # Index to the next unprocessed change in the +change_array+.
+    attr_accessor :current_index
+
+    # ID of the last cached change log record.
+    attr_accessor :current_id
+
+    # Array with all cached changes.
+    # Processed change log records are replaced with +nil+.
+    attr_accessor :change_array
+
+    # Tree (hash) structure for fast access to all cached changes.
+    # First level of tree:
+    # * key: table name
+    # * value: 2nd level tree
+    # 2nd level tree:
+    # * key: the change_key value of the according change log records.
+    # * value:
+    #   An array of according change log records (column_name => value hash).
+    #   Additional entry of each change log hash:
+    #   * key: 'array_index'
+    #   * value: index to the change log record in +change_array+
+    attr_accessor :change_tree
+
+    # Date of last update of the cache
+    attr_accessor :last_updated
+
+    # Initializes / resets the cache.
+    def init_cache
+      self.change_tree = {}
+      self.change_array = []
+      self.current_index = 0
+    end
+    private :init_cache
+
+    # Create a new change log record cache.
+    # * +session+: The current +Session+
+    # * +database+: Either :+left+ or :+right+
+    def initialize(session, database)
+      self.session = session
+      self.database = database
+      self.connection = session.send(database)
+
+      init_cache
+      self.current_id = -1
+      self.last_updated = 1.year.ago
+    end
+
+    # Updates the cache.
+    # Options is a hash determining when the update is actually executed:
+    # * :+expire_time+: cache is older than the given number of seconds
+    # * :+forced+: if +true+ update the cache even if not yet expired
+    def update(options = {:forced => false, :expire_time => 1})
+      return unless options[:forced] or Time.now - self.last_updated >= options[:expire_time]
+      
+      self.last_updated = Time.now
+
+      # First, let's use a LIMIT clause (via :row_buffer_size option) to verify
+      # if there are any pending changes.
+      # (If there are many pending changes, this is (at least with PostgreSQL)
+      # much faster.)
+      cursor = connection.select_cursor(
+        :table => change_log_table,
+        :from => {'id' => current_id},
+        :exclude_starting_row => true,
+        :row_buffer_size => 1
+      )
+      return unless cursor.next?
+
+      # Something is here. Let's actually load it.
+      cursor = connection.select_cursor(
+        :table => change_log_table,
+        :from => {'id' => current_id},
+        :exclude_starting_row => true,
+        :type_cast => true,
+        :row_buffer_size => session.configuration.options[:row_buffer_size]
+      )
+      while cursor.next?
+        change = cursor.next_row
+        self.current_id = change['id']
+        self.change_array << change
+        change['array_index'] = self.change_array.size - 1
+
+        table_change_tree = change_tree[change['change_table']] ||= {}
+        key_changes = table_change_tree[change['change_key']] ||= []
+        key_changes << change
+      end
+      cursor.clear
+    end
+
+    # Returns the creation time of the oldest unprocessed change log record.
+    def oldest_change_time
+      change = oldest_change
+      change['change_time'] if change
+    end
+
+    # Returns the oldest unprocessed change log record (column_name => value hash).
+    def oldest_change
+      update
+      oldest_change = nil
+      unless change_array.empty?
+        while (oldest_change = change_array[self.current_index]) == nil
+          self.current_index += 1
+        end
+      end
+      oldest_change
+    end
+
+    # Returns the specified change log record (column_name => value hash).
+    # * +change_table+: the name of the table that was changed
+    # * +change_key+: the change key of the modified record
+    def load(change_table, change_key)
+      update
+      change = nil
+      table_change_tree = change_tree[change_table]
+      if table_change_tree
+        key_changes = table_change_tree[change_key]
+        if key_changes
+          # get change object and delete from key_changes
+          change = key_changes.shift
+
+          # delete change from change_array
+          change_array[change['array_index']] = nil
+
+          # delete change from database
+          connection.execute "delete from #{change_log_table} where id = #{change['id']}"
+
+          # delete key_changes if empty
+          if key_changes.empty?
+            table_change_tree.delete change_key
+          end
+
+          # delete table_change_tree if empty
+          if table_change_tree.empty?
+            change_tree.delete change_table
+          end
+
+          # reset everything if no more changes remain
+          if change_tree.empty?
+            init_cache
+          end
+        end
+      end
+      change
+    end
+
+    # Returns the name of the change log table
+    def change_log_table
+      @change_log_table ||= "#{session.configuration.options[:rep_prefix]}_pending_changes"
+    end
+    private :change_log_table
+  end
+end

data/lib/rubyrep/noisy_connection.rb

@@ -0,0 +1,80 @@
+module RR
+
+  # Wraps an existing cursor.
+  # Purpose: send regular updates to the installed TaskSweeper
+  class NoisyCursor
+    # The original cusor
+    attr_accessor :org_cursor
+
+    # The installed task sweeper
+    attr_accessor :sweeper
+
+    # Create a new NoisyCursor.
+    # * cursor: the original cursor
+    # * sweeper: the target TaskSweeper
+    def initialize(cursor, sweeper)
+      self.org_cursor = cursor
+      self.sweeper = sweeper
+    end
+
+    # Delegate the uninteresting methods to the original cursor
+    def next?; org_cursor.next? end
+    def clear; org_cursor.clear end
+
+    # Returns the row as a column => value hash and moves the cursor to the next row.
+    def next_row
+      sweeper.ping
+      row = org_cursor.next_row
+      sweeper.ping
+      row
+    end
+  end
+
+  # Modifies ProxyConnections to send regular pings to an installed TaskSweeper
+  module NoisyConnection
+
+    # The installed TaskSweeper
+    attr_accessor :sweeper
+
+    # Modifies ProxyConnection#select_cursor to wrap the returned cursor
+    # into a NoisyCursor.
+    def select_cursor(options)
+      sweeper.ping
+      org_cursor = super
+      sweeper.ping
+      NoisyCursor.new(org_cursor, sweeper)
+    end
+
+    # Wraps ProxyConnection#insert_record to update the TaskSweeper
+    def insert_record(table, values)
+      sweeper.ping
+      result = super
+      sweeper.ping
+      result
+    end
+
+    # Wraps ProxyConnection#update_record to update the TaskSweeper
+    def update_record(table, values, org_key = nil)
+      sweeper.ping
+      result = super
+      sweeper.ping
+      result
+    end
+
+    # Wraps ProxyConnection#delete_record to update the TaskSweeper
+    def delete_record(table, values)
+      sweeper.ping
+      result = super
+      sweeper.ping
+      result
+    end
+
+    # Wraps ProxyConnection#commit_db_transaction to update the TaskSweeper
+    def commit_db_transaction
+      sweeper.ping
+      result = super
+      sweeper.ping
+      result
+    end
+  end
+end

data/lib/rubyrep/proxied_table_scan.rb

@@ -0,0 +1,171 @@
+module RR
+  
+  # Scans two tables for differences. Goes through a RubyRep Proxy to minimize network load.
+  # Doesn't have any reporting functionality by itself. 
+  # Instead ProxiedTableScan#run yields all the differences for the caller to do with as it pleases.
+  # Usage:
+  #   1. Create a new ProxiedTableScan object and hand it all necessary information
+  #   2. Call ProxiedTableScan#run to do the actual comparison
+  #   3. The block handed to ProxiedTableScan#run receives all differences
+  class ProxiedTableScan < TableScan
+    
+    # returns block size to use for table scanning
+    def block_size
+      @block_size ||= session.configuration.options_for_table(left_table)[:proxy_block_size]
+    end
+  
+    # Creates a new ProxiedTableScan instance
+    #   * session: a Session object representing the current database session
+    #   * left_table: name of the table in the left database
+    #   * right_table: name of the table in the right database. If not given, same like left_table
+    def initialize(session, left_table, right_table = nil)
+      raise "#{self.class.name} only works with proxied sessions" unless session.proxied?
+
+      super
+    end
+    
+    # Compares the specified left and right rows.
+    # +left_cursor+ and +right_cursor+ represent the according ProxyBlockCursor objects.
+    # Yields all identified differences with
+    # * diff_type
+    # * row
+    # #run described the yield parameters in detail.
+    def compare_blocks(left_block_cursor, right_block_cursor)
+      left_cursor = right_cursor = nil
+      
+      left_row_checksums = left_block_cursor.row_checksums
+      right_row_checksums = right_block_cursor.row_checksums
+
+      # phase 1: identify the different rows and store their primary keys
+      left_diff_rows = []
+      left_diff_checksums = []
+      right_diff_rows = []
+      right_diff_checksums = []
+      i = k = 0
+      while i < left_row_checksums.size or k < right_row_checksums.size
+        left_keys = i < left_row_checksums.size ? left_row_checksums[i][:row_keys] : nil
+        right_keys = k < right_row_checksums.size ? right_row_checksums[k][:row_keys] : nil
+        rank = rank_rows left_keys, right_keys
+        case rank
+        when -1
+          left_diff_rows << left_keys
+          left_diff_checksums << left_row_checksums[i][:checksum]
+          i += 1
+        when 1
+          right_diff_rows << right_keys
+          right_diff_checksums << right_row_checksums[k][:checksum]
+          k += 1
+        when 0
+          if left_row_checksums[i][:checksum] != right_row_checksums[k][:checksum]
+            left_diff_rows << left_keys
+            left_diff_checksums << left_row_checksums[i][:checksum]
+            right_diff_rows << right_keys
+            right_diff_checksums << right_row_checksums[k][:checksum]
+          end
+          i += 1
+          k += 1
+        end
+      end
+      
+      # retrieve possibly existing cached rows from the block cursors
+      left_row_cache = left_block_cursor.retrieve_row_cache left_diff_checksums
+      right_row_cache = right_block_cursor.retrieve_row_cache right_diff_checksums
+      
+      # builds arrays of row keys for rows that were not included in the hash
+      left_uncached_rows = []
+      left_diff_rows.each_with_index do |row, i|
+        left_uncached_rows << row unless left_row_cache[left_diff_checksums[i]]
+      end
+      right_uncached_rows = []
+      right_diff_rows.each_with_index do |row, i|
+        right_uncached_rows << row unless right_row_cache[right_diff_checksums[i]]
+      end
+
+      # phase 2: read all different rows and yield them
+      unless left_uncached_rows.empty?
+        left_cursor = session.left.create_cursor \
+          ProxyRowCursor, left_table, :row_keys => left_uncached_rows
+      end
+      unless right_uncached_rows.empty?
+        right_cursor = session.right.create_cursor \
+          ProxyRowCursor, right_table, :row_keys => right_uncached_rows  
+      end
+      i = k = 0
+      while i < left_diff_rows.size or k < right_diff_rows.size
+        rank = rank_rows left_diff_rows[i], right_diff_rows[k]
+        case rank
+        when -1
+          if left_row_cache.include? left_diff_checksums[i]
+            row = Marshal.load(left_row_cache[left_diff_checksums[i]])
+          else
+            row = left_cursor.next_row
+          end
+          yield :left, row
+          i += 1
+        when 1
+          if right_row_cache.include? right_diff_checksums[k]
+            row = Marshal.load(right_row_cache[right_diff_checksums[k]])
+          else
+            row = right_cursor.next_row
+          end
+          yield :right, row
+          k += 1
+        when 0
+          if left_row_cache.include? left_diff_checksums[i]
+            left_row = Marshal.load(left_row_cache[left_diff_checksums[i]])
+          else
+            left_row = left_cursor.next_row
+          end
+          if right_row_cache.include? right_diff_checksums[k]
+            right_row = Marshal.load(right_row_cache[right_diff_checksums[k]])
+          else
+            row = right_cursor.next_row
+          end
+          yield :conflict, [left_row, right_row]
+          i += 1
+          k += 1
+        end
+      end
+    ensure
+      session.left.destroy_cursor left_cursor if left_cursor
+      session.right.destroy_cursor right_cursor if right_cursor
+    end
+
+    # Runs the table scan.
+    # Calls the block for every found difference.
+    # Differences are yielded with 2 parameters
+    #   * type: describes the difference, either :left (row only in left table), :right (row only in right table) or :conflict
+    #   * row: for :left or :right cases a hash describing the row; for :conflict an array of left and right row
+    def run(&blck)
+      left_cursor = right_cursor = nil
+      left_cursor = session.left.create_cursor ProxyBlockCursor, left_table, 
+        :row_buffer_size => scan_options[:row_buffer_size]
+      right_cursor = session.right.create_cursor ProxyBlockCursor, right_table, 
+        :row_buffer_size => scan_options[:row_buffer_size]
+      update_progress 0 # ensures progress bar is printed even if there are no records
+      while left_cursor.next?
+        left_to, left_checksum, left_progress =
+          left_cursor.checksum :proxy_block_size => block_size
+        _ , right_checksum, right_progress =
+          right_cursor.checksum :max_row => left_to
+        combined_progress = left_progress + right_progress
+        if left_checksum != right_checksum
+          compare_blocks left_cursor, right_cursor do |type, row|
+            steps = type == :conflict ? 2 : 1
+            update_progress steps
+            combined_progress -= steps
+            yield type, row
+          end
+        end
+        update_progress combined_progress
+      end
+      while right_cursor.next?
+        update_progress 1
+        yield :right, right_cursor.next_row
+      end
+    ensure
+      session.left.destroy_cursor left_cursor if left_cursor
+      session.right.destroy_cursor right_cursor if right_cursor      
+    end
+  end
+end

data/lib/rubyrep/proxy_block_cursor.rb

@@ -0,0 +1,145 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
+
+require 'digest/sha1'
+
+require 'rubyrep'
+
+module RR
+  
+  # This class is used to scan a table in blocks.
+  # Calculates the checksums of the scanned blocks.
+  class ProxyBlockCursor < ProxyCursor
+    
+    include TableScanHelper
+    
+    # The current Digest
+    attr_accessor :digest
+    
+    # nil if the last run of the checksum method left no unprocessed row.
+    # Otherwise the left over row of that checksum run
+    attr_accessor :last_row
+    
+    # Returns an array of checksums for each encounters row.
+    # Each array element is a Hash with the following elements:
+    # * +:row_keys+: A primary key => value hash identifying the row
+    # * +:checksum+: the checksum for this row
+    attr_accessor :row_checksums
+    
+    # The maximum total size (in bytes) up to which rows will be cached
+    attr_accessor :max_row_cache_size
+    
+    # A byte counter of many bytes of row data have already been cached
+    attr_accessor :current_row_cache_size
+    
+    # A hash of cached rows consisting of row checksum => row dump pairs.
+    attr_accessor :row_cache
+        
+    # Creates a new cursor
+    # * session: the current proxy session
+    # * table: table_name
+    def initialize(session, table)
+      self.max_row_cache_size = 1000000 # this size should be sufficient as long as table doesn't contain blobs
+      super
+    end
+    
+    # Returns true if the current cursor has unprocessed rows
+    def next?
+      last_row != nil or cursor.next?
+    end
+    
+    # Returns the cursor's next row
+    def next_row
+      if self.last_row
+        row, self.last_row = self.last_row, nil
+      else
+        row = cursor.next_row
+      end
+      row
+    end
+    
+    # Returns a hash of row checksum => row dump pairs for the +checksums+ 
+    # in the provided array
+    def retrieve_row_cache(checksums)
+      row_dumps = {}
+      checksums.each do |checksum|
+        row_dumps[checksum] = row_cache[checksum] if row_cache.include? checksum
+      end      
+      row_dumps
+    end
+    
+    # Updates block / row checksums and row cache with the given +row+.
+    def update_checksum(row)
+      dump = Marshal.dump(row)
+      
+      # updates row checksum array
+      row_keys = row.reject {|key, | not primary_key_names.include? key}
+      checksum = Digest::SHA1.hexdigest(dump)
+      self.row_checksums << {:row_keys => row_keys, :checksum => checksum}
+
+      # update the row cache (unless maximum cache size limit has already been reached)
+      if current_row_cache_size + dump.size < max_row_cache_size
+        self.current_row_cache_size += dump.size
+        row_cache[checksum] = dump
+      end
+      
+      # update current total checksum
+      self.digest << dump
+    end
+    
+    # Reinitializes the row checksum array and the total checksum
+    def reset_checksum
+      self.row_checksums = []
+      self.current_row_cache_size = 0
+      self.row_cache = {}
+      self.digest = Digest::SHA1.new
+    end
+    
+    # Returns the current checksum
+    def current_checksum
+      self.digest.hexdigest
+    end
+    
+    # Calculates the checksum from the current row up to the row specified by options.
+    # options is a hash including either 
+    # * :+proxy_block_size+: The number of rows to scan.
+    # * :+max_row+: A row hash of primary key columns specifying the maximum record to scan.
+    # Returns multiple parameters:
+    # * last row read
+    # * checksum
+    # * number of processed records
+    def checksum(options = {})
+      reset_checksum
+      return_row = row = nil
+      row_count = 0
+
+      if options.include? :proxy_block_size
+        block_size = options[:proxy_block_size]
+        raise ":proxy_block_size must be greater than 0" unless block_size > 0
+        while row_count < block_size and next?
+          row = next_row
+          update_checksum(row)
+          row_count += 1
+        end
+        return_row = row
+      elsif options.include? :max_row
+        max_row = options[:max_row]
+        while next?
+          row = next_row
+          rank = rank_rows row, max_row
+          if rank > 0 
+            # row > max_row ==> save the current row and break off
+            self.last_row = row
+            break
+          end
+          row_count += 1
+          update_checksum(row)
+          return_row, row = row, nil
+        end  
+      else
+        raise "options must include either :proxy_block_size or :max_row"
+      end
+      return_keys = return_row.reject {|key, | not primary_key_names.include? key} if return_row
+      return return_keys, current_checksum, row_count
+    end
+  end
+end