rubyrep 1.0.0

data/lib/rubyrep/logged_change.rb

@@ -0,0 +1,326 @@
+module RR
+  
+  class Session
+
+    # Returns the +LoggedChangeLoader+ of the specified database.
+    # * database: either :+left+ or :+right+
+    def change_loader(database)
+      @change_loaders ||= {}
+      unless change_loader = @change_loaders[database]
+        change_loader = @change_loaders[database] = LoggedChangeLoader.new(self, database)
+      end
+      change_loader
+    end
+
+    # Forces an update of the change log cache
+    def reload_changes
+      change_loader(:left).update :forced => true
+      change_loader(:right).update :forced => true
+    end
+
+  end
+
+  # Caches the entries in the change log table
+  class LoggedChangeLoader
+
+    # The current +Session+.
+    attr_accessor :session
+
+    # The current +ProxyConnection+.
+    attr_accessor :connection
+
+    # Index to the next unprocessed change in the +change_array+.
+    attr_accessor :current_index
+
+    # ID of the last cached change log record.
+    attr_accessor :current_id
+
+    # Array with all cached changes.
+    # Processed change log records are replaced with +nil+.
+    attr_accessor :change_array
+
+    # Tree (hash) structure for fast access to all cached changes.
+    # First level of tree:
+    # * key: table name
+    # * value: 2nd level tree
+    # 2nd level tree:
+    # * key: the change_key value of the according change log records.
+    # * value:
+    #   The according change log record (column_name => value hash).
+    #   Additional entry of each change log hash:
+    #   * key: 'array_index'
+    #   * value: index to the change log record in +change_array+
+    attr_accessor :change_tree
+
+    # Date of last update of the cache
+    attr_accessor :last_updated
+
+    # Initializes / resets the cache.
+    def init_cache
+      self.change_tree = {}
+      self.change_array = []
+      self.current_index = 0
+    end
+    private :init_cache
+
+    # Create a new change log record cache.
+    # * +session+: The current +Session+
+    # * +database+: Either :+left+ or :+right+
+    def initialize(session, database)
+      self.session = session
+      self.connection = session.send(database)
+
+      init_cache
+      self.current_id = -1
+      self.last_updated = 1.year.ago
+    end
+
+    # Updates the cache.
+    # Options is a hash determining when the update is actually executed:
+    # * :+expire_time+: cache is older than the given number of seconds
+    # * :+forced+: if +true+ update the cache even if not yet expired
+    def update(options = {:forced => false, :expire_time => 1})
+      return unless options[:forced] or Time.now - self.last_updated >= options[:expire_time]
+      
+      self.last_updated = Time.now
+
+      cursor = connection.select_cursor(
+        :table => change_log_table,
+        :from => {'id' => current_id},
+        :exclude_starting_row => true,
+        :type_cast => true
+      )
+      while cursor.next?
+        change = cursor.next_row
+        self.current_id = change['id']
+        self.change_array << change
+        change['array_index'] = self.change_array.size - 1
+
+        table_change_tree = change_tree[change['change_table']] ||= {}
+        key_changes = table_change_tree[change['change_key']] ||= []
+        key_changes << change
+      end
+      cursor.clear
+    end
+
+    # Returns the creation time of the oldest unprocessed change log record.
+    def oldest_change_time
+      change = oldest_change
+      change['change_time'] if change
+    end
+
+    # Returns the oldest unprocessed change log record (column_name => value hash).
+    def oldest_change
+      update
+      oldest_change = nil
+      unless change_array.empty?
+        while (oldest_change = change_array[self.current_index]) == nil
+          self.current_index += 1
+        end
+      end
+      oldest_change
+    end
+
+    # Returns the specified change log record (column_name => value hash).
+    # * +change_table+: the name of the table that was changed
+    # * +change_key+: the change key of the modified record
+    def load(change_table, change_key)
+      update
+      change = nil
+      table_change_tree = change_tree[change_table]
+      if table_change_tree
+        key_changes = table_change_tree[change_key]
+        if key_changes
+          # get change object and delete from key_changes
+          change = key_changes.shift
+
+          # delete change from change_array
+          change_array[change['array_index']] = nil
+
+          # delete change from database
+          connection.execute "delete from #{change_log_table} where id = #{change['id']}"
+
+          # delete key_changes if empty
+          if key_changes.empty?
+            table_change_tree.delete change_key
+          end
+
+          # delete table_change_tree if empty
+          if table_change_tree.empty?
+            change_tree.delete change_table
+          end
+
+          # reset everything if no more changes remain
+          if change_tree.empty?
+            init_cache
+          end
+        end
+      end
+      change
+    end
+
+    # Returns the name of the change log table
+    def change_log_table
+      @change_log_table ||= "#{session.configuration.options[:rep_prefix]}_pending_changes"
+    end
+    private :change_log_table
+  end
+
+  # Describes a single logged record change.
+  # 
+  # Note:
+  # The change loading functionality depends on the current database session
+  # being executed in an open database transaction.
+  # Also at the end of change processing the transaction must be committed.
+  class LoggedChange
+
+    # The current Session
+    attr_accessor :session
+
+    # The database which was changed. Either :+left+ or :+right+.
+    attr_accessor :database
+
+    # The name of the changed table
+    attr_accessor :table
+
+    # When the first change to the record happened
+    attr_accessor :first_changed_at
+
+    # When the last change to the record happened
+    attr_accessor :last_changed_at
+
+    # Type of the change. Either :+insert+, :+update+ or :+delete+.
+    attr_accessor :type
+
+    # A column_name => value hash identifying the changed record
+    attr_accessor :key
+
+    # Only used for updates: a column_name => value hash of the original primary
+    # key of the updated record
+    attr_accessor :new_key
+
+    # Creates a new LoggedChange instance.
+    # * +session+: the current Session
+    # * +database+: either :+left+ or :+right+
+    def initialize(session, database)
+      self.session = session
+      self.database = database
+      self.type = :no_change
+    end
+
+    # A hash describing how the change state morph based on newly found change
+    # records.
+    # * key: String consisting of 2 letters
+    #   * first letter: describes current type change (nothing, insert, update, delete)
+    #   * second letter: the new change type as read of the change log table
+    # * value:
+    #   The resulting change type.
+    # [1]: such cases shouldn't happen. but just in case, choose the most
+    # sensible solution.
+    TYPE_CHANGES = {
+      'NI' => 'I',
+      'NU' => 'U',
+      'ND' => 'D',
+      'II' => 'I', # [1]
+      'IU' => 'I',
+      'ID' => 'N',
+      'UI' => 'U', # [1]
+      'UU' => 'U',
+      'UD' => 'D',
+      'DI' => 'U',
+      'DU' => 'U', # [1]
+      'DD' => 'D', # [1]
+    }
+
+    # A hash translating the short 1-letter types to the according symbols
+    SHORT_TYPES = {
+      'I' => :insert,
+      'U' => :update,
+      'D' => :delete,
+      'N' => :no_change
+    }
+    # A hash translating the symbold types to according 1 letter types
+    LONG_TYPES = SHORT_TYPES.invert
+
+    # Returns the configured key separator
+    def key_sep
+      @key_sep ||= session.configuration.options[:key_sep]
+    end
+
+    # Returns a column_name => value hash based on the provided +raw_key+ string
+    # (which is a string in the format as read directly from the change log table).
+    def key_to_hash(raw_key)
+      result = {}
+      #raw_key.split(key_sep).each_slice(2) {|a| result[a[0]] = a[1]}
+      raw_key.split(key_sep).each_slice(2) {|field_name, value| result[field_name] = value}
+      result
+    end
+
+    # Loads the change as per #table and #key. Works if the LoggedChange instance
+    # is totally new or was already loaded before.
+    def load
+      current_type = LONG_TYPES[type]
+
+      org_key = new_key || key
+      # change to key string as can be found in change log table
+      org_key = session.send(database).primary_key_names(table).map do |key_name|
+        "#{key_name}#{key_sep}#{org_key[key_name]}"
+      end.join(key_sep)
+      current_key = org_key
+
+      while change = session.change_loader(database).load(table, current_key)
+
+        new_type = change['change_type']
+        current_type = TYPE_CHANGES["#{current_type}#{new_type}"]
+
+        self.first_changed_at ||= change['change_time']
+        self.last_changed_at = change['change_time']
+
+        if change['change_type'] == 'U' and change['change_new_key'] != current_key
+          current_key = change['change_new_key']
+        end
+      end
+
+      self.type = SHORT_TYPES[current_type]
+      self.new_key = nil
+      if type == :update
+        self.key ||= key_to_hash(org_key)
+        self.new_key = key_to_hash(current_key)
+      else
+        self.key = key_to_hash(current_key)
+      end
+    end
+
+    # Loads the change with the specified key for the named +table+.
+    # * +table+: name of the table
+    # * +key+: a column_name => value hash for all primary key columns of the table
+    def load_specified(table, key)
+      self.table = table
+      self.key = key
+      load
+    end
+
+    # Returns the time of the oldest change. Returns +nil+ if there are no
+    # changes left.
+    def oldest_change_time
+      session.change_loader(database).oldest_change_time
+    end
+
+    # Loads the oldest available change
+    def load_oldest
+      begin
+        change = session.change_loader(database).oldest_change
+        break unless change
+        self.key = key_to_hash(change['change_key'])
+        self.table = change['change_table']
+        load
+      end until type != :no_change
+    end
+
+    # Prevents session from going into YAML output
+    def to_yaml_properties
+      instance_variables.sort.reject {|var_name| var_name == '@session'}
+    end
+
+  end
+end

data/lib/rubyrep/proxied_table_scan.rb

@@ -0,0 +1,171 @@
+module RR
+  
+  # Scans two tables for differences. Goes through a RubyRep Proxy to minimize network load.
+  # Doesn't have any reporting functionality by itself. 
+  # Instead ProxiedTableScan#run yields all the differences for the caller to do with as it pleases.
+  # Usage:
+  #   1. Create a new ProxiedTableScan object and hand it all necessary information
+  #   2. Call ProxiedTableScan#run to do the actual comparison
+  #   3. The block handed to ProxiedTableScan#run receives all differences
+  class ProxiedTableScan < TableScan
+    
+    # returns block size to use for table scanning
+    def block_size
+      @block_size ||= session.configuration.options_for_table(left_table)[:proxy_block_size]
+    end
+  
+    # Creates a new ProxiedTableScan instance
+    #   * session: a Session object representing the current database session
+    #   * left_table: name of the table in the left database
+    #   * right_table: name of the table in the right database. If not given, same like left_table
+    def initialize(session, left_table, right_table = nil)
+      raise "#{self.class.name} only works with proxied sessions" unless session.proxied?
+
+      super
+    end
+    
+    # Compares the specified left and right rows.
+    # +left_cursor+ and +right_cursor+ represent the according ProxyBlockCursor objects.
+    # Yields all identified differences with
+    # * diff_type
+    # * row
+    # #run described the yield parameters in detail.
+    def compare_blocks(left_block_cursor, right_block_cursor)
+      left_cursor = right_cursor = nil
+      
+      left_row_checksums = left_block_cursor.row_checksums
+      right_row_checksums = right_block_cursor.row_checksums
+
+      # phase 1: identify the different rows and store their primary keys
+      left_diff_rows = []
+      left_diff_checksums = []
+      right_diff_rows = []
+      right_diff_checksums = []
+      i = k = 0
+      while i < left_row_checksums.size or k < right_row_checksums.size
+        left_keys = i < left_row_checksums.size ? left_row_checksums[i][:row_keys] : nil
+        right_keys = k < right_row_checksums.size ? right_row_checksums[k][:row_keys] : nil
+        rank = rank_rows left_keys, right_keys
+        case rank
+        when -1
+          left_diff_rows << left_keys
+          left_diff_checksums << left_row_checksums[i][:checksum]
+          i += 1
+        when 1
+          right_diff_rows << right_keys
+          right_diff_checksums << right_row_checksums[k][:checksum]
+          k += 1
+        when 0
+          if left_row_checksums[i][:checksum] != right_row_checksums[k][:checksum]
+            left_diff_rows << left_keys
+            left_diff_checksums << left_row_checksums[i][:checksum]
+            right_diff_rows << right_keys
+            right_diff_checksums << right_row_checksums[k][:checksum]
+          end
+          i += 1
+          k += 1
+        end
+      end
+      
+      # retrieve possibly existing cached rows from the block cursors
+      left_row_cache = left_block_cursor.retrieve_row_cache left_diff_checksums
+      right_row_cache = right_block_cursor.retrieve_row_cache right_diff_checksums
+      
+      # builds arrays of row keys for rows that were not included in the hash
+      left_uncached_rows = []
+      left_diff_rows.each_with_index do |row, i|
+        left_uncached_rows << row unless left_row_cache[left_diff_checksums[i]]
+      end
+      right_uncached_rows = []
+      right_diff_rows.each_with_index do |row, i|
+        right_uncached_rows << row unless right_row_cache[right_diff_checksums[i]]
+      end
+
+      # phase 2: read all different rows and yield them
+      unless left_uncached_rows.empty?
+        left_cursor = session.left.create_cursor \
+          ProxyRowCursor, left_table, :row_keys => left_uncached_rows
+      end
+      unless right_uncached_rows.empty?
+        right_cursor = session.right.create_cursor \
+          ProxyRowCursor, right_table, :row_keys => right_uncached_rows  
+      end
+      i = k = 0
+      while i < left_diff_rows.size or k < right_diff_rows.size
+        rank = rank_rows left_diff_rows[i], right_diff_rows[k]
+        case rank
+        when -1
+          if left_row_cache.include? left_diff_checksums[i]
+            row = Marshal.load(left_row_cache[left_diff_checksums[i]])
+          else
+            row = left_cursor.next_row
+          end
+          yield :left, row
+          i += 1
+        when 1
+          if right_row_cache.include? right_diff_checksums[k]
+            row = Marshal.load(right_row_cache[right_diff_checksums[k]])
+          else
+            row = right_cursor.next_row
+          end
+          yield :right, row
+          k += 1
+        when 0
+          if left_row_cache.include? left_diff_checksums[i]
+            left_row = Marshal.load(left_row_cache[left_diff_checksums[i]])
+          else
+            left_row = left_cursor.next_row
+          end
+          if right_row_cache.include? right_diff_checksums[k]
+            right_row = Marshal.load(right_row_cache[right_diff_checksums[k]])
+          else
+            row = right_cursor.next_row
+          end
+          yield :conflict, [left_row, right_row]
+          i += 1
+          k += 1
+        end
+      end
+    ensure
+      session.left.destroy_cursor left_cursor if left_cursor
+      session.right.destroy_cursor right_cursor if right_cursor
+    end
+
+    # Runs the table scan.
+    # Calls the block for every found difference.
+    # Differences are yielded with 2 parameters
+    #   * type: describes the difference, either :left (row only in left table), :right (row only in right table) or :conflict
+    #   * row: for :left or :right cases a hash describing the row; for :conflict an array of left and right row
+    def run(&blck)
+      left_cursor = right_cursor = nil
+      left_cursor = session.left.create_cursor ProxyBlockCursor, left_table, 
+        :row_buffer_size => scan_options[:row_buffer_size]
+      right_cursor = session.right.create_cursor ProxyBlockCursor, right_table, 
+        :row_buffer_size => scan_options[:row_buffer_size]
+      update_progress 0 # ensures progress bar is printed even if there are no records
+      while left_cursor.next?
+        left_to, left_checksum, left_progress =
+          left_cursor.checksum :proxy_block_size => block_size
+        _ , right_checksum, right_progress =
+          right_cursor.checksum :max_row => left_to
+        combined_progress = left_progress + right_progress
+        if left_checksum != right_checksum
+          compare_blocks left_cursor, right_cursor do |type, row|
+            steps = type == :conflict ? 2 : 1
+            update_progress steps
+            combined_progress -= steps
+            yield type, row
+          end
+        end
+        update_progress combined_progress
+      end
+      while right_cursor.next?
+        update_progress 1
+        yield :right, right_cursor.next_row
+      end
+    ensure
+      session.left.destroy_cursor left_cursor if left_cursor
+      session.right.destroy_cursor right_cursor if right_cursor      
+    end
+  end
+end

data/lib/rubyrep/proxy_block_cursor.rb

@@ -0,0 +1,145 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/..'
+
+require 'digest/sha1'
+
+require 'rubyrep'
+
+module RR
+  
+  # This class is used to scan a table in blocks.
+  # Calculates the checksums of the scanned blocks.
+  class ProxyBlockCursor < ProxyCursor
+    
+    include TableScanHelper
+    
+    # The current Digest
+    attr_accessor :digest
+    
+    # nil if the last run of the checksum method left no unprocessed row.
+    # Otherwise the left over row of that checksum run
+    attr_accessor :last_row
+    
+    # Returns an array of checksums for each encounters row.
+    # Each array element is a Hash with the following elements:
+    # * +:row_keys+: A primary key => value hash identifying the row
+    # * +:checksum+: the checksum for this row
+    attr_accessor :row_checksums
+    
+    # The maximum total size (in bytes) up to which rows will be cached
+    attr_accessor :max_row_cache_size
+    
+    # A byte counter of many bytes of row data have already been cached
+    attr_accessor :current_row_cache_size
+    
+    # A hash of cached rows consisting of row checksum => row dump pairs.
+    attr_accessor :row_cache
+        
+    # Creates a new cursor
+    # * session: the current proxy session
+    # * table: table_name
+    def initialize(session, table)
+      self.max_row_cache_size = 1000000 # this size should be sufficient as long as table doesn't contain blobs
+      super
+    end
+    
+    # Returns true if the current cursor has unprocessed rows
+    def next?
+      last_row != nil or cursor.next?
+    end
+    
+    # Returns the cursor's next row
+    def next_row
+      if self.last_row
+        row, self.last_row = self.last_row, nil
+      else
+        row = cursor.next_row
+      end
+      row
+    end
+    
+    # Returns a hash of row checksum => row dump pairs for the +checksums+ 
+    # in the provided array
+    def retrieve_row_cache(checksums)
+      row_dumps = {}
+      checksums.each do |checksum|
+        row_dumps[checksum] = row_cache[checksum] if row_cache.include? checksum
+      end      
+      row_dumps
+    end
+    
+    # Updates block / row checksums and row cache with the given +row+.
+    def update_checksum(row)
+      dump = Marshal.dump(row)
+      
+      # updates row checksum array
+      row_keys = row.reject {|key, | not primary_key_names.include? key}
+      checksum = Digest::SHA1.hexdigest(dump)
+      self.row_checksums << {:row_keys => row_keys, :checksum => checksum}
+
+      # update the row cache (unless maximum cache size limit has already been reached)
+      if current_row_cache_size + dump.size < max_row_cache_size
+        self.current_row_cache_size += dump.size
+        row_cache[checksum] = dump
+      end
+      
+      # update current total checksum
+      self.digest << dump
+    end
+    
+    # Reinitializes the row checksum array and the total checksum
+    def reset_checksum
+      self.row_checksums = []
+      self.current_row_cache_size = 0
+      self.row_cache = {}
+      self.digest = Digest::SHA1.new
+    end
+    
+    # Returns the current checksum
+    def current_checksum
+      self.digest.hexdigest
+    end
+    
+    # Calculates the checksum from the current row up to the row specified by options.
+    # options is a hash including either 
+    # * :+proxy_block_size+: The number of rows to scan.
+    # * :+max_row+: A row hash of primary key columns specifying the maximum record to scan.
+    # Returns multiple parameters:
+    # * last row read
+    # * checksum
+    # * number of processed records
+    def checksum(options = {})
+      reset_checksum
+      return_row = row = nil
+      row_count = 0
+
+      if options.include? :proxy_block_size
+        block_size = options[:proxy_block_size]
+        raise ":proxy_block_size must be greater than 0" unless block_size > 0
+        while row_count < block_size and next?
+          row = next_row
+          update_checksum(row)
+          row_count += 1
+        end
+        return_row = row
+      elsif options.include? :max_row
+        max_row = options[:max_row]
+        while next?
+          row = next_row
+          rank = rank_rows row, max_row
+          if rank > 0 
+            # row > max_row ==> save the current row and break off
+            self.last_row = row
+            break
+          end
+          row_count += 1
+          update_checksum(row)
+          return_row, row = row, nil
+        end  
+      else
+        raise "options must include either :proxy_block_size or :max_row"
+      end
+      return_keys = return_row.reject {|key, | not primary_key_names.include? key} if return_row
+      return return_keys, current_checksum, row_count
+    end
+  end
+end