perobs 4.1.0 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 4da7810046e0984b00152ab2205ebb2e2d364b94
-  data.tar.gz: 74c48af35b674713b222d0dff1b7f0c3ef070b45
+SHA256:
+  metadata.gz: 7790ee42586bb2b8fca115f93ed277c4a8057f7a7027b356baea7b066da953e5
+  data.tar.gz: 110e0710a84ef544a4874cf868ec1662dfc900077d1894b24f53bcdaeaeeed34
 SHA512:
-  metadata.gz: 5746476c388361dee6ac06094e7545e86ecb0569368c089e21429e4e6c3dbf92d8db5e7854249d7214c162f15a629306092dd0ca8bfbceb85eda7c08df95cf90
-  data.tar.gz: a495a6127f26bdc7326f1bd7ea3671ff9add499d0b455a9334c884c23a2960ee963eedb8d2bcb5c3f257af9dde4895bebd458a96e7c079dbd0752028e9200614
+  metadata.gz: d95d845c7e8bd183f53b60369415bde86cd766c224bcbc2c52c870c60542a786f359a63eecc4e8829055cee6a1674bc952277749da183432b4b3abae7536efbb
+  data.tar.gz: 5fa1712fb01118d955d86396aec87b319c75856f6602ccf1a19f475a3dc64dc65a540e35271e6f2c6c7ee6a10c5cb03da30d43945a0dc264c7b0e48f575269d1

data/lib/perobs/BTree.rb

@@ -53,8 +53,8 @@ module PEROBS
       @name = name
       @progressmeter = progressmeter
 
-      unless order > 2
-        PEROBS.log.fatal "BTree order must be larger than 2, not #{order}"
+      unless order > 4
+        PEROBS.log.fatal "BTree order must be larger than 4, not #{order}"
       end
       unless order % 2 == 1
         PEROBS.log.fatal "BTree order must be an uneven number, not #{order}"
@@ -70,7 +70,7 @@ module PEROBS
       @nodes.register_custom_data('first_leaf')
       @nodes.register_custom_data('last_leaf')
       @nodes.register_custom_data('btree_size')
-      @node_cache = PersistentObjectCache.new(16384, 5000, BTreeNode, self)
+      @node_cache = PersistentObjectCache.new(2**16, -1, BTreeNode, self)
       @root = @first_leaf = @last_leaf = nil
       @size = 0
 
@@ -117,12 +117,10 @@ module PEROBS
 
     # Close the tree file.
     def close
-
-      def val_perc(value, total)
-        "#{value} (#{(value.to_f / total*100.0).to_i}%)"
-      end
-
       sync
+      PEROBS.log.info "BTree file #{@name} has currently " +
+        "#{@nodes.total_entries} used entries and #{@nodes.total_spaces} " +
+        "unused entries"
       @nodes.close
       @root = nil
     end
@@ -165,21 +163,34 @@ module PEROBS
       return false unless @nodes.check
 
       entries = 0
-      res = true
+      stats = nil
       @progressmeter.start('Checking index structure', @size) do |pm|
-        res = @root.check do |k, v|
+        stats = @root.check do |k, v|
           pm.update(entries += 1)
           block_given? ? yield(k, v) : true
         end
       end
 
+      return false unless stats
+
       unless entries == @size
         PEROBS.log.error "The BTree size (#{@size}) and the number of " +
           "found entries (#{entries}) don't match"
         return false
       end
+      unless stats.nodes_count == @nodes.total_entries
+        PEROBS.log.error "The BTree nodes count (#{stats.nodes_count}) and " +
+          "the number of entries in the nodes file (#{@nodes.total_entries}) " +
+          "don't match"
+          return false
+      end
+      PEROBS.log.info "Statistics for the BTree #{@name}: " +
+        "Number of nodes: #{stats.nodes_count}; " +
+        "Branch depth: #{stats.branch_depth}; " +
+        "Number of leave nodes: #{stats.leave_nodes}; " +
+        "Number of leaves: #{stats.leaves}"
 
-      res
+      !stats.nil?
     end
 
     # Register a new node as root node of the tree.
@@ -208,8 +219,10 @@ module PEROBS
     # @param key [Integer] Unique key
     # @param value [Integer] value
     def insert(key, value)
-      @size += 1 if @root.insert(key, value)
-      @node_cache.flush
+      if @root.insert(key, value)
+        @size += 1
+        @node_cache.flush
+      end
     end
 
     # Retrieve the value associated with the given key. If no entry was found,
@@ -220,6 +233,13 @@ module PEROBS
       @root.get(key)
     end
 
+    # Either return the key/value pair that exactly matches the key or a
+    # key/value pair that has a key that is at least min_miss_increment larger
+    # than the key.
+    def get_best_match(key, min_miss_increment)
+      @root.get_best_match(key, min_miss_increment)
+    end
+
     # Find and remove the value associated with the given key. If no entry was
     # found, return nil, otherwise the found value.
     def remove(key)

data/lib/perobs/BTreeBlob.rb

@@ -2,7 +2,7 @@
 #
 # = BTreeBlob.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2019 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -144,11 +144,12 @@ module PEROBS
 
     # Remove all entries from the index that have not been marked.
     # @return [Array] List of deleted object IDs.
-    def delete_unmarked_entries
+    def delete_unmarked_entries(&block)
       deleted_ids = []
       # First remove the entry from the hash table.
       @entries_by_id.delete_if do |id, e|
         if e[MARKED] == 0
+          yield(id) if block_given?
           deleted_ids << id
           true
         else

data/lib/perobs/BTreeDB.rb

@@ -2,7 +2,8 @@
 #
 # = BTreeDB.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016, 2018 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2018, 2019
+# by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -159,9 +160,9 @@ module PEROBS
     # Permanently delete all objects that have not been marked. Those are
     # orphaned and are no longer referenced by any actively used object.
     # @return [Array] List of IDs that have been removed from the DB.
-    def delete_unmarked_objects
+    def delete_unmarked_objects(&block)
       deleted_ids = []
-      each_blob { |blob| deleted_ids += blob.delete_unmarked_entries }
+      each_blob { |blob| deleted_ids += blob.delete_unmarked_entries(&block) }
       deleted_ids
     end
 

data/lib/perobs/BTreeNode.rb

@@ -39,6 +39,8 @@ module PEROBS
   # mark a node as leaf or branch node.
   class BTreeNode
 
+    Stats = Struct.new(:branch_depth, :nodes_count, :leave_nodes, :leaves)
+
     attr_reader :node_address, :parent, :is_leaf, :next_sibling, :prev_sibling,
       :keys, :values, :children
 
@@ -225,6 +227,7 @@ module PEROBS
         else
           # Descend into the right child node to add the value to.
           node = node.children[node.search_key_index(key)]
+          node = node.get_node if node
         end
       end
 
@@ -249,6 +252,62 @@ module PEROBS
 
         # Descend into the right child node to continue the search.
         node = node.children[i]
+        node = node.get_node if node
+      end
+
+      PEROBS.log.fatal "Could not find proper node to get from while " +
+        "looking for key #{key}"
+    end
+
+    # Return the key/value pair that matches the given key or the next larger
+    # key/value pair with a key that is at least as large as key +
+    # min_miss_increment.
+    # @param key [Integer] key to search for
+    # @param min_miss_increment [Integer] minimum required key increment in
+    #        case an exact key match could not be found
+    # @return [Integer or nil] value that matches the key
+    def get_best_match(key, min_miss_increment)
+      node = self
+
+      while node do
+        # Find index of the entry that best fits the key.
+        i = node.search_key_index(key)
+        if node.is_leaf
+          # This is a leaf node. Check if there is an exact match for the
+          # given key.
+          if node.keys[i] == key
+            # Return the corresponding value/value pair.
+            return [ key, node.values[i] ]
+          else
+            # No exact key match. Now search the larger keys for the first
+            # that is at least key + min_miss_increment large.
+            keys = node.keys
+            keys_length = keys.length
+            while node
+              if i >= keys_length
+                # We've reached the end of a node. Continue search in next
+                # sibling.
+                return nil unless (node = node.next_sibling)
+                node = node.get_node
+                keys = node.keys
+                keys_length = keys.length
+                i = -1
+              elsif keys[i] >= key + min_miss_increment
+                # We've found a key that fits the critera. Return the
+                # corresponding key/value pair.
+                return [ keys[i], node.values[i] ]
+              end
+
+              i += 1
+            end
+
+            return nil
+          end
+        end
+
+        # Descend into the right child node to continue the search.
+        node = node.children[i]
+        node = node.get_node if node
       end
 
       PEROBS.log.fatal "Could not find proper node to get from while " +
@@ -277,6 +336,7 @@ module PEROBS
 
         # Descend into the right child node to continue the search.
         node = node.children[i]
+        node = node.get_node if node
       end
 
       PEROBS.log.fatal 'Could not find proper node to remove from'
@@ -310,22 +370,6 @@ module PEROBS
       @parent
     end
 
-    def merge_node(upper_sibling, parent_index)
-      if upper_sibling == self
-        PEROBS.log.fatal "Cannot merge node @#{@node_address} with self"
-      end
-      unless upper_sibling.is_leaf
-        insert_element(@parent.keys[parent_index], upper_sibling.children[0])
-      end
-      upper_sibling.copy_elements(0, self, @keys.size, upper_sibling.keys.size)
-      if (@next_sibling = link(upper_sibling.next_sibling))
-        @next_sibling.prev_sibling = link(self)
-      end
-      @tree.delete_node(upper_sibling.node_address)
-
-      @parent.remove_element(parent_index)
-    end
-
     # Insert the given value or child into the current node using the key as
     # index.
     # @param key [Integer] key to address the value or child
@@ -428,13 +472,8 @@ module PEROBS
         end
       end
 
-      if @parent.nil? && @children.length == 1
-        # If the node just below the root only has one child it will become
-        # the new root node.
-        new_root = @children.first
-        new_root.parent = nil
-        @tree.set_root(new_root)
-      end
+      # Delete the node from the cache and backing store.
+      @tree.delete_node(node.node_address)
     end
 
     def merge_with_leaf_node(node)
@@ -561,36 +600,8 @@ module PEROBS
     # @param key [Integer] key to search for
     # @return [Integer] Index of the matching key or the insert position.
     def search_key_index(key)
-      # Handle special case for empty keys list.
-      return 0 if @keys.empty?
-
-      # Keys are unique and always sorted. Use a binary search to find the
-      # index that fits the given key.
-      li = pi = 0
-      ui = @keys.size - 1
-      while li <= ui
-        # The pivot element is always in the middle between the lower and upper
-        # index.
-        pi = li + (ui - li) / 2
-
-        if key < @keys[pi]
-          # The pivot element is smaller than the key. Set the upper index to
-          # the pivot index.
-          ui = pi - 1
-        elsif key > @keys[pi]
-          # The pivot element is larger than the key. Set the lower index to
-          # the pivot index.
-          li = pi + 1
-        else
-          # We've found an exact match. For leaf nodes return the found index.
-          # For branch nodes we have to add one to the index since the larger
-          # child is the right one.
-          return @is_leaf ? pi : pi + 1
-        end
-      end
-      # No exact match was found. For the insert operaton we need to return
-      # the index of the first key that is larger than the given key.
-      @keys[pi] < key ? pi + 1 : pi
+      (@is_leaf ? @keys.bsearch_index { |x| x >= key } :
+                  @keys.bsearch_index { |x| x > key }) || @keys.length
     end
 
     # Iterate over all the key/value pairs in this node and all sub-nodes.
@@ -641,25 +652,28 @@ module PEROBS
     # Check consistency of the node and all subsequent nodes. In case an error
     # is found, a message is logged and false is returned.
     # @yield [key, value]
-    # @return [Boolean] true if tree has no errors
+    # @return [nil or Hash] nil in case of errors or a hash with some
+    #         statistical information about the tree
     def check
-      branch_depth = nil
+      stats = Stats.new(nil, 0, 0, 0)
 
       traverse do |node, position, stack|
         if position == 0
+          stats.nodes_count += 1
           if node.parent
             # After a split the nodes will only have half the maximum keys.
             # For branch nodes one of the split nodes will have even 1 key
             # less as this will become the branch key in a parent node.
             if node.keys.size < min_keys - (node.is_leaf ? 0 : 1)
               node.error "BTreeNode #{node.node_address} has too few keys"
-              return false
+              return nil
             end
           end
 
           if node.keys.size > @tree.order
             node.error "BTreeNode must not have more then #{@tree.order} " +
               "keys, but has #{node.keys.size} keys"
+            return nil
           end
 
           last_key = nil
@@ -667,79 +681,82 @@ module PEROBS
             if last_key && key < last_key
               node.error "Keys are not increasing monotoneously: " +
                 "#{node.keys.inspect}"
-              return false
+              return nil
             end
             last_key = key
           end
 
           if node.is_leaf
-            if branch_depth
-              unless branch_depth == stack.size
+            if stats.branch_depth
+              unless stats.branch_depth == node.tree_level
                 node.error "All leaf nodes must have same distance from root "
-                return false
+                return nil
               end
             else
-              branch_depth = stack.size
+              stats.branch_depth = node.tree_level
             end
             if node.prev_sibling.nil? && @tree.first_leaf != node
               node.error "Leaf node #{node.node_address} has no previous " +
                 "sibling but is not the first leaf of the tree"
-              return false
+              return nil
             end
             if node.next_sibling.nil? && @tree.last_leaf != node
               node.error "Leaf node #{node.node_address} has no next sibling " +
                 "but is not the last leaf of the tree"
-              return false
+              return nil
             end
             unless node.keys.size == node.values.size
               node.error "Key count (#{node.keys.size}) and value " +
                 "count (#{node.values.size}) don't match"
-                return false
+                return nil
             end
             unless node.children.empty?
               node.error "@children must be nil for a leaf node"
-              return false
+              return nil
             end
+
+            stats.leave_nodes += 1
+            stats.leaves += node.keys.length
           else
             unless node.values.empty?
               node.error "@values must be nil for a branch node"
-              return false
+              return nil
             end
             unless node.children.size == node.keys.size + 1
               node.error "Key count (#{node.keys.size}) must be one " +
                 "less than children count (#{node.children.size})"
-                return false
+                return nil
             end
             node.children.each_with_index do |child, i|
               unless child.is_a?(BTreeNodeLink)
                 node.error "Child #{i} is of class #{child.class} " +
                   "instead of BTreeNodeLink"
-                return false
+                return nil
               end
               unless child.parent.is_a?(BTreeNodeLink)
                 node.error "Parent reference of child #{i} is of class " +
                   "#{child.parent.class} instead of BTreeNodeLink"
-                return false
+                return nil
               end
               if child == node
                 node.error "Child #{i} points to self"
-                return false
+                return nil
               end
               if stack.include?(child)
                 node.error "Child #{i} points to ancester node"
-                return false
+                return nil
               end
               unless child.parent == node
                 node.error "Child #{i} does not have parent pointing " +
                   "to this node"
-                return false
+                return nil
               end
               if i > 0
                 unless node.children[i - 1].next_sibling == child
                   node.error "next_sibling of node " +
                     "#{node.children[i - 1].node_address} " +
                     "must point to node #{child.node_address}"
-                  return false
+                  return nil
                 end
               end
               if i < node.children.length - 1
@@ -747,7 +764,7 @@ module PEROBS
                   node.error "prev_sibling of node " +
                     "#{node.children[i + 1].node_address} " +
                     "must point to node #{child.node_address}"
-                  return false
+                  return nil
                 end
               end
             end
@@ -761,24 +778,24 @@ module PEROBS
               node.error "Child #{node.children[index].node_address} " +
                 "has too large key #{node.children[index].keys.last}. " +
                 "Must be smaller than #{node.keys[index]}."
-              return false
+              return nil
             end
             unless node.children[position].keys.first >= node.keys[index]
               node.error "Child #{node.children[position].node_address} " +
                 "has too small key #{node.children[position].keys.first}. " +
                 "Must be larger than or equal to #{node.keys[index]}."
-              return false
+              return nil
             end
           else
             if block_given?
               # If a block was given, call this block with the key and value.
-              return false unless yield(node.keys[index], node.values[index])
+              return nil unless yield(node.keys[index], node.values[index])
             end
           end
         end
       end
 
-      true
+      stats
     end
 
     def is_top?
@@ -834,6 +851,7 @@ module PEROBS
 
         str = (is_last_child ? '   ' : '  |') + str
         node = node.parent
+        node = node.get_node if node
       end
 
       str
@@ -871,6 +889,17 @@ module PEROBS
       s
     end
 
+    def tree_level
+      level = 1
+      node = self
+      while (node = node.parent)
+        level += 1
+      end
+
+      level
+    end
+
+
     def error(msg)
       PEROBS.log.error "Error in BTreeNode @#{@node_address}: #{msg}"
     end