perobs 3.0.1 → 4.3.0

data/lib/perobs/BTreeBlob.rb

@@ -2,7 +2,7 @@
 #
 # = BTreeBlob.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2019 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -33,7 +33,7 @@ require 'perobs/RobustFile'
 module PEROBS
 
   # This class manages the usage of the data blobs in the corresponding
-  # HashedBlobsDB object.
+  # BTreeDB object.
   class BTreeBlob
 
     # Magic number used for index files.
@@ -65,7 +65,7 @@ module PEROBS
     end
 
     # Write the given bytes with the given ID into the DB.
-    # @param id [Fixnum or Bignum] ID
+    # @param id [Integer] ID
     # @param raw [String] sequence of bytes
     def write_object(id, raw)
       if @entries.length > @btreedb.max_blob_size
@@ -87,7 +87,7 @@ module PEROBS
     end
 
     # Read the entry for the given ID and return it as bytes.
-    # @param id [Fixnum or Bignum] ID
+    # @param id [Integer] ID
     # @return [String] sequence of bytes or nil if ID is unknown
     def read_object(id)
       return nil unless (index_entry = find(id))
@@ -95,7 +95,7 @@ module PEROBS
     end
 
     # Find the data for the object with given id.
-    # @param id [Fixnum or Bignum] Object ID
+    # @param id [Integer] Object ID
     # @return [Array] Returns an Array that represents the index entry for the
     #         given object.
     def find(id)
@@ -109,7 +109,7 @@ module PEROBS
     end
 
     # Set a mark on the entry with the given ID.
-    # @param id [Fixnum or Bignum] ID of the entry
+    # @param id [Integer] ID of the entry
     def mark(id)
       found = false
       @entries.each do |entry|
@@ -129,7 +129,7 @@ module PEROBS
     end
 
     # Check if the entry for a given ID is marked.
-    # @param id [Fixnum or Bignum] ID of the entry
+    # @param id [Integer] ID of the entry
     # @param ignore_errors [Boolean] If set to true no errors will be raised
     #        for non-existing objects.
     # @return [TrueClass or FalseClass] true if marked, false otherwise
@@ -144,11 +144,12 @@ module PEROBS
 
     # Remove all entries from the index that have not been marked.
     # @return [Array] List of deleted object IDs.
-    def delete_unmarked_entries
+    def delete_unmarked_entries(&block)
       deleted_ids = []
       # First remove the entry from the hash table.
       @entries_by_id.delete_if do |id, e|
         if e[MARKED] == 0
+          yield(id) if block_given?
           deleted_ids << id
           true
         else
@@ -205,8 +206,8 @@ module PEROBS
 
     # Write a string of bytes into the file at the given address.
     # @param raw [String] bytes to write
-    # @param address [Fixnum] offset in the file
-    # @return [Fixnum] number of bytes written
+    # @param address [Integer] offset in the file
+    # @return [Integer] number of bytes written
     def write_to_blobs_file(raw, address)
       begin
         File.write(@blobs_file_name, raw, address)
@@ -236,9 +237,9 @@ module PEROBS
 
     # Reserve the bytes needed for the specified number of bytes with the
     # given ID.
-    # @param id [Fixnum or Bignum] ID of the entry
-    # @param bytes [Fixnum] number of bytes for this entry
-    # @return [Fixnum] the start address of the reserved blob
+    # @param id [Integer] ID of the entry
+    # @param bytes [Integer] number of bytes for this entry
+    # @return [Integer] the start address of the reserved blob
     def reserve_bytes(id, bytes, crc32)
       # index of first blob after the last seen entry
       end_of_last_entry = 0

data/lib/perobs/BTreeDB.rb

@@ -2,7 +2,8 @@
 #
 # = BTreeDB.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2018, 2019
+# by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -58,7 +59,7 @@ module PEROBS
     #                         nodes. The insert/find/delete time grows
     #                         linearly with the size.
     def initialize(db_name, options = {})
-      super(options[:serializer] || :json)
+      super(options)
 
       @db_dir = db_name
       # Create the database directory if it doesn't exist yet.
@@ -103,7 +104,7 @@ module PEROBS
     end
 
     # Return true if the object with given ID exists
-    # @param id [Fixnum or Bignum]
+    # @param id [Integer]
     def include?(id)
       !(blob = find_blob(id)).nil? && !blob.find(id).nil?
     end
@@ -143,7 +144,7 @@ module PEROBS
     end
 
     # Load the given object from the filesystem.
-    # @param id [Fixnum or Bignum] object ID
+    # @param id [Integer] object ID
     # @return [Hash] Object as defined by PEROBS::ObjectBase or nil if ID does
     #         not exist
     def get_object(id)
@@ -159,20 +160,20 @@ module PEROBS
     # Permanently delete all objects that have not been marked. Those are
     # orphaned and are no longer referenced by any actively used object.
     # @return [Array] List of IDs that have been removed from the DB.
-    def delete_unmarked_objects
+    def delete_unmarked_objects(&block)
       deleted_ids = []
-      each_blob { |blob| deleted_ids += blob.delete_unmarked_entries }
+      each_blob { |blob| deleted_ids += blob.delete_unmarked_entries(&block) }
       deleted_ids
     end
 
     # Mark an object.
-    # @param id [Fixnum or Bignum] ID of the object to mark
+    # @param id [Integer] ID of the object to mark
     def mark(id)
       (blob = find_blob(id)) && blob.mark(id)
     end
 
     # Check if the object is marked.
-    # @param id [Fixnum or Bignum] ID of the object to check
+    # @param id [Integer] ID of the object to check
     # @param ignore_errors [Boolean] If set to true no errors will be raised
     #        for non-existing objects.
     def is_marked?(id, ignore_errors = false)
@@ -189,7 +190,7 @@ module PEROBS
     end
 
     # Check if the stored object is syntactically correct.
-    # @param id [Fixnum/Bignum] Object ID
+    # @param id [Integer] Object ID
     # @param repair [TrueClass/FalseClass] True if an repair attempt should be
     #        made.
     # @return [TrueClass/FalseClass] True if the object is OK, otherwise
@@ -208,7 +209,7 @@ module PEROBS
     # Store the given serialized object into the cluster files. This method is
     # for internal use only!
     # @param raw [String] Serialized Object as defined by PEROBS::ObjectBase
-    # @param id [Fixnum or Bignum] Object ID
+    # @param id [Integer] Object ID
     def put_raw_object(raw, id)
       find_blob(id, true).write_object(id, raw)
     end

data/lib/perobs/BTreeNode.rb

@@ -39,8 +39,10 @@ module PEROBS
   # mark a node as leaf or branch node.
   class BTreeNode
 
-    attr_reader :node_address, :parent, :is_leaf, :keys, :values, :children
-    attr_accessor :dirty
+    Stats = Struct.new(:branch_depth, :nodes_count, :leave_nodes, :leaves)
+
+    attr_reader :node_address, :parent, :is_leaf, :next_sibling, :prev_sibling,
+      :keys, :values, :children
 
     # Create a new BTreeNode object for the given tree with the given parent
     # or recreate the node with the given node_address from the backing store.
@@ -49,58 +51,162 @@ module PEROBS
     # restore the node.
     # @param tree [BTree] The tree this node is part of
     # @param parent [BTreeNode] reference to parent node
+    # @param prev_sibling [BTreeNode] reference to previous sibling node
+    # @param next_sibling [BTreeNode] reference to next sibling node
     # @param node_address [Integer] the address of the node to read from the
     #        backing store
     # @param is_leaf [Boolean] true if the node should be a leaf node, false
     #        if not
-    def initialize(tree, parent = nil, node_address = nil, is_leaf = true)
+    def initialize(tree, node_address = nil, parent = nil, is_leaf = true,
+                   prev_sibling = nil, next_sibling = nil,
+                   keys = nil, values = nil, children = nil)
       @tree = tree
-      @parent = nil
       if node_address == 0
         PEROBS.log.fatal "Node address may not be 0"
       end
       @node_address = node_address
-      @keys = []
+      @parent = link(parent)
+      @prev_sibling = link(prev_sibling)
+      @next_sibling = link(next_sibling)
+      @keys = keys || []
       if (@is_leaf = is_leaf)
-        @values = []
+        @values = values || []
+        @children = nil
       else
-        @children = []
+        @children = children || []
+        @values = nil
       end
+    end
 
-      if node_address
-        unless node_address.is_a?(Integer)
-          PEROBS.log.fatal "node_address is not Integer: #{node_address.class}"
-        end
+    # Create a new SpaceTreeNode. This method should be used for the creation
+    # of new nodes instead of calling the constructor directly.
+    # @param tree [BTree] The tree the new node should belong to
+    # @param parent [BTreeNode] The parent node
+    # @param is_leaf [Boolean] True if the node has no children, false
+    #        otherwise
+    # @param prev_sibling [BTreeNode] reference to previous sibling node
+    # @param next_sibling [BTreeNode] reference to next sibling node
+    def BTreeNode::create(tree, parent = nil, is_leaf = true,
+                          prev_sibling = nil, next_sibling = nil)
+      unless parent.nil? || parent.is_a?(BTreeNode) ||
+             parent.is_a?(BTreeNodeLink)
+        PEROBS.log.fatal "Parent node must be a BTreeNode but is of class " +
+          "#{parent.class}"
+      end
 
-        # This must be an existing node. Try to read it and fill the instance
-        # variables.
-        unless read_node
-          PEROBS.log.fatal "SpaceTree node at address #{node_address} " +
-            "does not exist"
-        end
+      address = tree.nodes.free_address
+      node = BTreeNode.new(tree, address, parent, is_leaf, prev_sibling,
+                           next_sibling)
+      # This is a new node. Make sure the data is written to the file.
+      tree.node_cache.insert(node)
+
+      # Insert the newly created node into the existing node chain.
+      if (node.prev_sibling = prev_sibling)
+        node.prev_sibling.next_sibling = BTreeNodeLink.new(tree, node)
+      end
+      if (node.next_sibling = next_sibling)
+        node.next_sibling.prev_sibling = BTreeNodeLink.new(tree, node)
+      end
+
+      BTreeNodeLink.new(tree, node)
+    end
+
+    # Restore a node from the backing store at the given address and tree.
+    # @param tree [BTree] The tree the node belongs to
+    # @param address [Integer] The address in the blob file.
+    def BTreeNode::load(tree, address, unused = nil)
+      unless address.is_a?(Integer)
+        PEROBS.log.fatal "address is not Integer: #{address.class}"
+      end
+
+      unless (bytes = tree.nodes.retrieve_blob(address))
+        PEROBS.log.fatal "SpaceTree node at address #{address} " +
+          "does not exist"
+      end
+
+      unless Zlib::crc32(bytes) != 0
+        PEROBS.log.fatal "Checksum failure in BTreeNode entry @#{address}"
+      end
+      ary = bytes.unpack(BTreeNode::node_bytes_format(tree))
+      # Read is_leaf
+      if ary[0] != 0 && ary[0] != 1
+        PEROBS.log.fatal "First byte of a BTreeNode entry must be 0 or 1"
+      end
+      is_leaf = ary[0] == 0 ? false : true
+      # This is the number of keys this node has.
+      key_count = ary[1]
+      data_count = ary[2]
+      # Read the parent node address
+      parent = ary[3] == 0 ? nil : BTreeNodeLink.new(tree, ary[3])
+      prev_sibling = ary[4] == 0 ? nil : BTreeNodeLink.new(tree, ary[4])
+      next_sibling = ary[5] == 0 ? nil : BTreeNodeLink.new(tree, ary[5])
+      # Read the keys
+      keys = ary[6, key_count]
+
+      children = nil
+      values = nil
+      if is_leaf
+        # Read the values
+        values = ary[6 + tree.order, data_count]
       else
-        unless parent.nil? || parent.is_a?(BTreeNode) ||
-               parent.is_a?(BTreeNodeLink)
-          PEROBS.log.fatal "Parent node must be a BTreeNode but is of class " +
-            "#{parent.class}"
+        # Read the child addresses
+        children = []
+        data_count.times do |i|
+          child_address = ary[6 + tree.order + i]
+          unless child_address > 0
+            PEROBS.log.fatal "Child address must be larger than 0"
+          end
+          children << BTreeNodeLink.new(tree, child_address)
         end
-
-        # This is a new node. Make sure the data is written to the file.
-        @node_address = @tree.nodes.free_address
-        self.parent = parent
       end
+
+      node = BTreeNode.new(tree, address, parent, is_leaf,
+                           prev_sibling, next_sibling, keys, values,
+                           children)
+      tree.node_cache.insert(node, false)
+
+      node
+    end
+
+    # This is a wrapper around BTreeNode::load() that returns a BTreeNodeLink
+    # instead of the actual node.
+    # @param tree [BTree] The tree the node belongs to
+    # @param address [Integer] The address in the blob file.
+    # @return [BTreeNodeLink] Link to loaded noded
+    def BTreeNode::load_and_link(tree, address)
+      BTreeNodeLink.new(tree, BTreeNode::load(tree, address))
+    end
+
+
+    # @return [String] The format used for String.pack.
+    def BTreeNode::node_bytes_format(tree)
+      # This does not include the 4 bytes for the CRC32 checksum
+      "CSSQQQQ#{tree.order}Q#{tree.order + 1}"
     end
 
+    # @return [Integer] The number of bytes needed to store a node.
     def BTreeNode::node_bytes(order)
       1 + # is_leaf
       2 + # actual key count
       2 + # actual value or children count (aka data count)
       8 + # parent address
+      8 + # previous sibling address
+      8 + # next sibling address
       8 * order + # keys
       8 * (order + 1) + # values or child addresses
       4 # CRC32 checksum
     end
 
+    # Save the node into the blob file.
+    def save
+      write_node
+    end
+
+    # The node address uniquely identifies a BTreeNode.
+    def uid
+      @node_address
+    end
+
     # Insert or replace the given value by using the key as unique address.
     # @param key [Integer] Unique key to retrieve the value
     # @param value [Integer] value to insert
@@ -117,11 +223,11 @@ module PEROBS
 
         # Once we have reached a leaf node we can insert or replace the value.
         if node.is_leaf
-          node.insert_element(key, value)
-          return
+          return node.insert_element(key, value)
         else
           # Descend into the right child node to add the value to.
           node = node.children[node.search_key_index(key)]
+          node = node.get_node if node
         end
       end
 
@@ -146,6 +252,62 @@ module PEROBS
 
         # Descend into the right child node to continue the search.
         node = node.children[i]
+        node = node.get_node if node
+      end
+
+      PEROBS.log.fatal "Could not find proper node to get from while " +
+        "looking for key #{key}"
+    end
+
+    # Return the key/value pair that matches the given key or the next larger
+    # key/value pair with a key that is at least as large as key +
+    # min_miss_increment.
+    # @param key [Integer] key to search for
+    # @param min_miss_increment [Integer] minimum required key increment in
+    #        case an exact key match could not be found
+    # @return [Integer or nil] value that matches the key
+    def get_best_match(key, min_miss_increment)
+      node = self
+
+      while node do
+        # Find index of the entry that best fits the key.
+        i = node.search_key_index(key)
+        if node.is_leaf
+          # This is a leaf node. Check if there is an exact match for the
+          # given key.
+          if node.keys[i] == key
+            # Return the corresponding value/value pair.
+            return [ key, node.values[i] ]
+          else
+            # No exact key match. Now search the larger keys for the first
+            # that is at least key + min_miss_increment large.
+            keys = node.keys
+            keys_length = keys.length
+            while node
+              if i >= keys_length
+                # We've reached the end of a node. Continue search in next
+                # sibling.
+                return nil unless (node = node.next_sibling)
+                node = node.get_node
+                keys = node.keys
+                keys_length = keys.length
+                i = -1
+              elsif keys[i] >= key + min_miss_increment
+                # We've found a key that fits the critera. Return the
+                # corresponding key/value pair.
+                return [ keys[i], node.values[i] ]
+              end
+
+              i += 1
+            end
+
+            return nil
+          end
+        end
+
+        # Descend into the right child node to continue the search.
+        node = node.children[i]
+        node = node.get_node if node
       end
 
       PEROBS.log.fatal "Could not find proper node to get from while " +
@@ -174,6 +336,7 @@ module PEROBS
 
         # Descend into the right child node to continue the search.
         node = node.children[i]
+        node = node.get_node if node
       end
 
       PEROBS.log.fatal 'Could not find proper node to remove from'
@@ -186,18 +349,19 @@ module PEROBS
     def split_node
       unless @parent
         # The node is the root node. We need to create a parent node first.
-        self.parent = @tree.new_node(nil, nil, false)
+        self.parent = link(BTreeNode::create(@tree, nil, false))
         @parent.set_child(0, self)
         @tree.set_root(@parent)
       end
 
       # Create the new sibling that will take the 2nd half of the
       # node content.
-      sibling = @tree.new_node(@parent, nil, @is_leaf)
+      sibling = BTreeNode::create(@tree, @parent, @is_leaf, link(self),
+                                  @next_sibling)
       # Determine the index of the middle element that gets moved to the
       # parent. The order must be an uneven number, so adding 1 will get us
       # the middle element.
-      mid = @tree.order / 2 + 1
+      mid = @tree.order / 2
       # Insert the middle element key into the parent node
       @parent.insert_element(@keys[mid], sibling)
       copy_elements(mid + (@is_leaf ? 0 : 1), sibling)
@@ -206,30 +370,17 @@ module PEROBS
       @parent
     end
 
-    def merge_node(upper_sibling, parent_index)
-      if upper_sibling == self
-        PEROBS.log.fatal "Cannot merge node @#{@node_address} with self"
-      end
-      unless upper_sibling.is_leaf
-        insert_element(@parent.keys[parent_index], upper_sibling.children[0])
-      end
-      upper_sibling.copy_elements(0, self, @keys.size, upper_sibling.keys.size)
-      @tree.delete_node(upper_sibling.node_address)
-
-      @parent.remove_element(parent_index)
-    end
-
     # Insert the given value or child into the current node using the key as
     # index.
     # @param key [Integer] key to address the value or child
     # @param value_or_child [Integer or BTreeNode] value or BTreeNode
     #        reference
+    # @return true for insert, false for overwrite
     def insert_element(key, value_or_child)
       if @keys.size >= @tree.order
         PEROBS.log.fatal "Cannot insert into a full BTreeNode"
       end
 
-      mark_as_modified
       i = search_key_index(key)
       if @keys[i] == key
         # Overwrite existing entries
@@ -237,58 +388,132 @@ module PEROBS
         if is_leaf
           @values[i] = value_or_child
         else
-          @children[i + 1] = BTreeNodeLink.new(@tree, value_or_child)
+          @children[i + 1] = link(value_or_child)
         end
+        @tree.node_cache.insert(self)
+
+        return false
       else
         # Create a new entry
         @keys.insert(i, key)
         if is_leaf
           @values.insert(i, value_or_child)
         else
-          @children.insert(i + 1, BTreeNodeLink.new(@tree, value_or_child))
+          @children.insert(i + 1, link(value_or_child))
         end
+        @tree.node_cache.insert(self)
+
+        return true
       end
     end
 
     # Remove the element at the given index.
     def remove_element(index)
-      # We need this key to find the link in the parent node.
-      first_key = @keys[0]
-      removed_value = nil
-
-      mark_as_modified
       # Delete the key at the specified index.
-      unless @keys.delete_at(index)
-        PEROBS.log.fatal "Could not remove element #{index} from BTreeNode " +
+      unless (key = @keys.delete_at(index))
+        PEROBS.log.fatal "Could not remove element #{index} from BigTreeNode " +
           "@#{@node_address}"
       end
-      if @is_leaf
-        # For leaf nodes, also delete the corresponding value.
-        removed_value = @values.delete_at(index)
+      update_branch_key(key) if index == 0
+
+      # Delete the corresponding value.
+      removed_value = @values.delete_at(index)
+      @tree.node_cache.insert(self)
+
+      if @keys.length < min_keys
+        if @prev_sibling && @prev_sibling.parent == @parent
+          borrow_from_previous_sibling(@prev_sibling) ||
+            @prev_sibling.merge_with_leaf_node(self)
+        elsif @next_sibling && @next_sibling.parent == @parent
+          borrow_from_next_sibling(@next_sibling) ||
+            merge_with_leaf_node(@next_sibling)
+        elsif @parent
+          PEROBS.log.fatal "Cannot not find adjecent leaf siblings"
+        end
+      end
+
+      # The merge has potentially invalidated this node. After this method has
+      # been called this copy of the node should no longer be used.
+      removed_value
+    end
+
+    def remove_child(node)
+      unless (index = search_node_index(node))
+        PEROBS.log.fatal "Cannot remove child #{node.node_address} " +
+          "from node #{@node_address}"
+      end
+
+      @tree.node_cache.insert(self)
+      if index == 0
+        # Removing the first child is a bit more complicated as the
+        # corresponding branch key is in a parent node.
+        key = @keys.shift
+        update_branch_key(key)
       else
-        # The corresponding child has can be found at 1 index higher.
-        @children.delete_at(index + 1)
+        # For all other children we can just remove the corresponding key.
+        @keys.delete_at(index - 1)
+      end
+
+      # Remove the child node link.
+      child = @children.delete_at(index)
+      # Unlink the neighbouring siblings from the child
+      child.prev_sibling.next_sibling = child.next_sibling if child.prev_sibling
+      child.next_sibling.prev_sibling = child.prev_sibling if child.next_sibling
+
+      if @keys.length < min_keys
+        # The node has become too small. Try borrowing a node from an adjecent
+        # sibling or merge with an adjecent node.
+        if @prev_sibling && @prev_sibling.parent == @parent
+          borrow_from_previous_sibling(@prev_sibling) ||
+            @prev_sibling.merge_with_branch_node(self)
+        elsif @next_sibling && @next_sibling.parent == @parent
+          borrow_from_next_sibling(@next_sibling) ||
+            merge_with_branch_node(@next_sibling)
+        end
       end
 
-      # Find the lower and upper siblings and the index of the key for this
-      # node in the parent node.
-      lower_sibling, upper_sibling, parent_index =
-        find_closest_siblings(first_key)
+      # Delete the node from the cache and backing store.
+      @tree.delete_node(node.node_address)
+    end
 
-      if lower_sibling &&
-         lower_sibling.keys.size + @keys.size < @tree.order
-        lower_sibling.merge_node(self, parent_index - 1)
-      elsif upper_sibling &&
-            @keys.size + upper_sibling.keys.size < @tree.order
-        merge_node(upper_sibling, parent_index)
+    def merge_with_leaf_node(node)
+      if @keys.length + node.keys.length > @tree.order
+        PEROBS.log.fatal "Leaf nodes are too big to merge"
       end
 
-      # The merge has potentially invalidated this node. After this method has
-      # been called this copy of the node should no longer be used.
-      removed_value
+      @keys += node.keys
+      @values += node.values
+      @tree.node_cache.insert(self)
+
+      node.parent.remove_child(node)
+    end
+
+    def merge_with_branch_node(node)
+      if @keys.length + 1 + node.keys.length > @tree.order
+        PEROBS.log.fatal "Branch nodes are too big to merge"
+      end
+
+      index = @parent.search_node_index(node) - 1
+      @keys << @parent.keys[index]
+      @keys += node.keys
+      node.children.each { |c| c.parent = link(self) }
+      @children += node.children
+      @tree.node_cache.insert(self)
+
+      node.parent.remove_child(node)
+    end
+
+    def search_node_index(node)
+      index = search_key_index(node.keys.first)
+      unless @children[index] == node
+        raise RuntimeError, "Child at index #{index} is not the requested node"
+      end
+
+      index
     end
 
     def copy_elements(src_idx, dest_node, dst_idx = 0, count = nil)
+      dest_node = dest_node.get_node
       unless count
         count = @tree.order - src_idx
       end
@@ -301,7 +526,6 @@ module PEROBS
       end
 
       dest_node.keys[dst_idx, count] = @keys[src_idx, count]
-      dest_node.dirty = true
       if @is_leaf
         # For leaves we copy the keys and corresponding values.
         dest_node.values[dst_idx, count] = @values[src_idx, count]
@@ -313,31 +537,61 @@ module PEROBS
           dest_node.set_child(dst_idx + i, @children[src_idx + i])
         end
       end
+      @tree.node_cache.insert(dest_node)
     end
 
     def parent=(p)
-      @parent = p ? BTreeNodeLink.new(@tree, p) : nil
-      mark_as_modified
+      @parent = p
+      @tree.node_cache.insert(self)
+
+      p
+    end
+
+    def prev_sibling=(node)
+      @prev_sibling = node
+      if node.nil? && @is_leaf
+        # If this node is a leaf node without a previous sibling we need to
+        # register it as the first leaf node.
+        @tree.set_first_leaf(BTreeNodeLink.new(@tree, self))
+      end
+
+      @tree.node_cache.insert(self)
+
+      node
+    end
+
+    def next_sibling=(node)
+      @next_sibling = node
+      @tree.node_cache.insert(self)
+      if node.nil? && @is_leaf
+        # If this node is a leaf node without a next sibling we need to
+        # register it as the last leaf node.
+        @tree.set_last_leaf(BTreeNodeLink.new(@tree, self))
+      end
+
+      node
     end
 
     def set_child(index, child)
       if child
-        @children[index] = BTreeNodeLink.new(@tree, child)
-        @children[index].parent = self
+        @children[index] = link(child)
+        @children[index].parent = link(self)
       else
         @children[index] = nil
       end
-      mark_as_modified
+      @tree.node_cache.insert(self)
+
+      child
     end
 
     def trim(idx)
-      mark_as_modified
-      @keys = @keys[0..idx - 1]
+      @keys.slice!(idx, @keys.length - idx)
       if @is_leaf
-        @values = @values[0..idx - 1]
+        @values.slice!(idx, @values.length - idx)
       else
-        @children = @children[0..idx]
+        @children.slice!(idx + 1, @children.length - idx - 1)
       end
+      @tree.node_cache.insert(self)
     end
 
     # Search the keys of the node that fits the given key. The result is
@@ -346,36 +600,8 @@ module PEROBS
     # @param key [Integer] key to search for
     # @return [Integer] Index of the matching key or the insert position.
     def search_key_index(key)
-      # Handle special case for empty keys list.
-      return 0 if @keys.empty?
-
-      # Keys are unique and always sorted. Use a binary search to find the
-      # index that fits the given key.
-      li = pi = 0
-      ui = @keys.size - 1
-      while li <= ui
-        # The pivot element is always in the middle between the lower and upper
-        # index.
-        pi = li + (ui - li) / 2
-
-        if key < @keys[pi]
-          # The pivot element is smaller than the key. Set the upper index to
-          # the pivot index.
-          ui = pi - 1
-        elsif key > @keys[pi]
-          # The pivot element is larger than the key. Set the lower index to
-          # the pivot index.
-          li = pi + 1
-        else
-          # We've found an exact match. For leaf nodes return the found index.
-          # For branch nodes we have to add one to the index since the larger
-          # child is the right one.
-          return @is_leaf ? pi : pi + 1
-        end
-      end
-      # No exact match was found. For the insert operaton we need to return
-      # the index of the first key that is larger than the given key.
-      @keys[pi] < key ? pi + 1 : pi
+      (@is_leaf ? @keys.bsearch_index { |x| x >= key } :
+                  @keys.bsearch_index { |x| x > key }) || @keys.length
     end
 
     # Iterate over all the key/value pairs in this node and all sub-nodes.
@@ -426,17 +652,33 @@ module PEROBS
     # Check consistency of the node and all subsequent nodes. In case an error
     # is found, a message is logged and false is returned.
     # @yield [key, value]
-    # @return [Boolean] true if tree has no errors
-    def check
+    # @return [nil or Hash] nil in case of errors or a hash with some
+    #         statistical information about the tree
+    def check(&block)
+      stats = Stats.new(nil, 0, 0, 0)
+
       traverse do |node, position, stack|
         if position == 0
-          if node.parent && node.keys.size < 1
-            node.error "BTreeNode must have at least one entry"
-            return false
+          stats.nodes_count += 1
+          if node.parent
+            unless node.parent.is_a?(BTreeNodeLink)
+              node.error "parent is a #{node.parent.class} instead of a " +
+                "BTreeNodeLink"
+              return nil
+            end
+            # After a split the nodes will only have half the maximum keys.
+            # For branch nodes one of the split nodes will have even 1 key
+            # less as this will become the branch key in a parent node.
+            if node.keys.size < min_keys - (node.is_leaf ? 0 : 1)
+              node.error "BTreeNode #{node.node_address} has too few keys"
+              return nil
+            end
           end
+
           if node.keys.size > @tree.order
             node.error "BTreeNode must not have more then #{@tree.order} " +
               "keys, but has #{node.keys.size} keys"
+            return nil
           end
 
           last_key = nil
@@ -444,45 +686,101 @@ module PEROBS
             if last_key && key < last_key
               node.error "Keys are not increasing monotoneously: " +
                 "#{node.keys.inspect}"
-              return false
+              return nil
             end
+            last_key = key
           end
 
           if node.is_leaf
+            if stats.branch_depth
+              unless stats.branch_depth == node.tree_level
+                node.error "All leaf nodes must have same distance from root "
+                return nil
+              end
+            else
+              stats.branch_depth = node.tree_level
+            end
+            if node.prev_sibling && !node.prev_sibling.is_a?(BTreeNodeLink)
+              node.error "prev_sibling is a #{node.prev_sibling.class} " +
+                "instead of a BTreeNodeLink"
+              return nil
+            end
+            if node.next_sibling && !node.next_sibling.is_a?(BTreeNodeLink)
+              node.error "next_sibling is a #{node.next_sibling.class} " +
+                "instead of a BTreeNodeLink"
+              return nil
+            end
+            if node.prev_sibling.nil? && @tree.first_leaf != node
+              node.error "Leaf node #{node.node_address} has no previous " +
+                "sibling but is not the first leaf of the tree"
+              return nil
+            end
+            if node.next_sibling.nil? && @tree.last_leaf != node
+              node.error "Leaf node #{node.node_address} has no next sibling " +
+                "but is not the last leaf of the tree"
+              return nil
+            end
             unless node.keys.size == node.values.size
               node.error "Key count (#{node.keys.size}) and value " +
                 "count (#{node.values.size}) don't match"
-                return false
+              return nil
+            end
+            unless node.children.nil?
+              node.error "@children must be nil for a leaf node"
+              return nil
             end
+
+            stats.leave_nodes += 1
+            stats.leaves += node.keys.length
           else
-            unless node.keys.size == node.children.size - 1
+            unless node.values.nil?
+              node.error "@values must be nil for a branch node"
+              return nil
+            end
+            unless node.children.size == node.keys.size + 1
               node.error "Key count (#{node.keys.size}) must be one " +
                 "less than children count (#{node.children.size})"
-                return false
+              return nil
             end
             node.children.each_with_index do |child, i|
               unless child.is_a?(BTreeNodeLink)
                 node.error "Child #{i} is of class #{child.class} " +
                   "instead of BTreeNodeLink"
-                return false
+                return nil
               end
               unless child.parent.is_a?(BTreeNodeLink)
                 node.error "Parent reference of child #{i} is of class " +
-                  "#{child.class} instead of BTreeNodeLink"
-                return false
+                  "#{child.parent.class} instead of BTreeNodeLink"
+                return nil
               end
-              if child.node_address == node.node_address
+              if child == node
                 node.error "Child #{i} points to self"
-                return false
+                return nil
               end
               if stack.include?(child)
                 node.error "Child #{i} points to ancester node"
-                return false
+                return nil
               end
               unless child.parent == node
                 node.error "Child #{i} does not have parent pointing " +
                   "to this node"
-                return false
+                return nil
+              end
+              if i > 0
+                unless node.children[i - 1].next_sibling == child
+                  node.error "next_sibling of node " +
+                    "#{node.children[i - 1].node_address} " +
+                    "must point to node #{child.node_address}"
+                  return nil
+                end
+              end
+              if i < node.children.length - 1
+                unless child == node.children[i + 1].prev_sibling
+                  node.error "prev_sibling of node " +
+                    "#{node.children[i + 1].node_address} " +
+                    "must point to node #{child.node_address}"
+                  return nil
+                end
               end
             end
           end
@@ -495,25 +793,26 @@ module PEROBS
               node.error "Child #{node.children[index].node_address} " +
                 "has too large key #{node.children[index].keys.last}. " +
                 "Must be smaller than #{node.keys[index]}."
-              return false
+              return nil
             end
-            unless node.children[position].keys.first >=
-                   node.keys[index]
+            unless node.children[position].keys.first >= node.keys[index]
               node.error "Child #{node.children[position].node_address} " +
                 "has too small key #{node.children[position].keys.first}. " +
                 "Must be larger than or equal to #{node.keys[index]}."
-              return false
+              return nil
             end
           else
             if block_given?
               # If a block was given, call this block with the key and value.
-              return false unless yield(node.keys[index], node.values[index])
+              unless yield(node.keys[index], node.values[index])
+                return nil
+              end
             end
           end
         end
       end
 
-      true
+      stats
     end
 
     def is_top?
@@ -569,6 +868,7 @@ module PEROBS
 
         str = (is_last_child ? '   ' : '  |') + str
         node = node.parent
+        node = node.get_node if node
       end
 
       str
@@ -588,23 +888,47 @@ module PEROBS
           s += ' ^@'
         end
       end
+      if @prev_sibling
+        begin
+          s += " <#{@prev_sibling.node_address}"
+        rescue
+          s += ' <@'
+        end
+      end
+      if @next_sibling
+        begin
+          s += " >#{@next_sibling.node_address}"
+        rescue
+          s += ' >@'
+        end
+      end
 
       s
     end
 
+    def tree_level
+      level = 1
+      node = self
+      while (node = node.parent)
+        level += 1
+      end
+
+      level
+    end
+
+
     def error(msg)
-      PEROBS.log.error "Error in BTreeNode @#{@node_address}: #{msg}\n" +
-        @tree.to_s
+      PEROBS.log.error "Error in BTreeNode @#{@node_address}: #{msg}"
     end
 
     def write_node
-      return unless @dirty
-
       ary = [
         @is_leaf ? 1 : 0,
         @keys.size,
         @is_leaf ? @values.size : @children.size,
-        @parent ? @parent.node_address : 0
+        @parent ? @parent.node_address : 0,
+        @prev_sibling ? @prev_sibling.node_address : 0,
+        @next_sibling ? @next_sibling.node_address : 0
       ] + @keys + ::Array.new(@tree.order - @keys.size, 0)
 
       if @is_leaf
@@ -620,84 +944,114 @@ module PEROBS
         ary += @children.map{ |c| c.node_address } +
           ::Array.new(@tree.order + 1 - @children.size, 0)
       end
-      bytes = ary.pack(node_bytes_format)
+      bytes = ary.pack(BTreeNode::node_bytes_format(@tree))
       bytes += [ Zlib::crc32(bytes) ].pack('L')
       @tree.nodes.store_blob(@node_address, bytes)
-      @dirty = false
-    end
-
-    def mark_as_modified
-      @tree.mark_node_as_modified(self)
-      @dirty = true
     end
 
     private
 
-    def read_node
-      return false unless (bytes = @tree.nodes.retrieve_blob(@node_address))
+    def min_keys
+      @tree.order / 2
+    end
 
-      unless Zlib::crc32(bytes) != 0
-        PEROBS.log.error "Checksum failure in BTreeNode entry @#{node_address}"
-        return false
-      end
-      ary = bytes.unpack(node_bytes_format)
-      # Read is_leaf
-      if ary[0] != 0 && ary[0] != 1
-        PEROBS.log.error "First byte of a BTreeNode entry must be 0 or 1"
-        return false
-      end
-      @is_leaf = ary[0] == 0 ? false : true
-      # This is the number of keys this node has.
-      key_count = ary[1]
-      data_count = ary[2]
-      # Read the parent node address
-      @parent = ary[3] == 0 ? nil : BTreeNodeLink.new(@tree, ary[3])
-      # Read the keys
-      @keys = ary[4, key_count]
+    def link(node)
+      return nil if node.nil?
 
-      if @is_leaf
-        # Read the values
-        @values = ary[4 + @tree.order, data_count]
+      if node.is_a?(BTreeNodeLink)
+        return node
+      elsif node.is_a?(BTreeNode) || node.is_a?(Integer)
+        return BTreeNodeLink.new(@tree, node)
       else
-        # Read the child addresses
-        @children = []
-        data_count.times do |i|
-          address = ary[4 + @tree.order + i]
-          unless address > 0
-            PEROBS.log.fatal "Child address must not be 0"
-          end
-          @children << BTreeNodeLink.new(@tree, address)
-        end
+        PEROBS.log.fatal "Node link must be a BTreeNode, not a #{node.class}"
       end
+    end
 
-      @dirty = false
+    # Try to borrow an element from the preceding sibling.
+    # @return [True or False] True if an element was borrowed, false
+    #         otherwise.
+    def borrow_from_previous_sibling(prev_node)
+      if prev_node.keys.length - 1 > min_keys
+        index = @parent.search_node_index(self) - 1
+
+        @tree.node_cache.insert(self)
+        @tree.node_cache.insert(prev_node.get_node)
+        @tree.node_cache.insert(@parent.get_node)
+        if @is_leaf
+          # Move the last key of the previous node to the front of this node
+          @keys.unshift(prev_node.keys.pop)
+          # Register the new lead key of this node with its parent
+          @parent.keys[index] = @keys.first
+          # Move the last value of the previous node to the front of this node
+          @values.unshift(prev_node.values.pop)
+        else
+          # For branch nodes the branch key will be the borrowed key.
+          @keys.unshift(@parent.keys[index])
+          # And the last key of the previous key will become the new branch
+          # key for this node.
+          @parent.keys[index] = prev_node.keys.pop
+          # Move the last child of the previous node to the front of this node
+          @children.unshift(node = prev_node.children.pop)
+          node.parent = link(self)
+        end
 
-      true
+        return true
+      end
+
+      false
     end
 
-    def node_bytes_format
-      # This does not include the 4 bytes for the CRC32 checksum
-      "CSSQQ#{@tree.order}Q#{@tree.order + 1}"
+    # Try to borrow an element from the next sibling.
+    # @return [True or False] True if an element was borrowed, false
+    #         otherwise.
+    def borrow_from_next_sibling(next_node)
+      if next_node.keys.length - 1 > min_keys
+        # The next sibling now has a new lead key that requires the branch key
+        # to be updated in the parent node.
+        index = next_node.parent.search_node_index(next_node) - 1
+
+        @tree.node_cache.insert(self)
+        @tree.node_cache.insert(next_node.get_node)
+        @tree.node_cache.insert(next_node.parent.get_node)
+        if @is_leaf
+          # Move the first key of the next node to the end of the this node
+          @keys << next_node.keys.shift
+          # Register the new lead key of next_node with its parent
+          next_node.parent.keys[index] = next_node.keys.first
+          # Move the first value of the next node to the end of this node
+          @values << next_node.values.shift
+        else
+          # For branch nodes we need to get the lead key from the parent of
+          # next_node.
+          @keys << next_node.parent.keys[index]
+          # The old lead key of next_node becomes the branch key in the parent
+          # of next_node. And the keys of next_node are shifted.
+          next_node.parent.keys[index] = next_node.keys.shift
+          # Move the first child of the next node to the end of this node
+          @children << (node = next_node.children.shift)
+          node.parent = link(self)
+        end
+
+        return true
+      end
+
+      false
     end
 
-    def find_closest_siblings(key)
-      # The root node has no siblings.
-      return [ nil, nil, nil ] unless @parent
+    def update_branch_key(old_key)
+      new_key = @keys.first
+      return unless (node = @parent)
 
-      parent_index = @parent.search_key_index(key)
-      unless @parent.children[parent_index] == self
-        PEROBS.log.fatal "Failed to find self in parent"
+      while node
+        if (index = node.keys.index(old_key))
+          node.keys[index] = new_key
+          @tree.node_cache.insert(node.get_node)
+          return
+        end
+        node = node.parent
       end
-      # The child that corresponds to the key at parent_index has an index of
-      # parent_index + 1! The lower_sibling has an child index of
-      # parent_index and the upper sibling has a child index of parent_index +
-      # 2.
-      lower_sibling = parent_index < 1 ?
-        nil : @parent.children[parent_index - 1]
-      upper_sibling = parent_index >= (@parent.children.size - 1) ?
-        nil : @parent.children[parent_index + 1]
 
-      [ lower_sibling, upper_sibling, parent_index ]
+      # The smallest element has no branch key.
     end
 
   end