perobs 4.0.0 → 4.4.0

data/lib/perobs/SpaceManager.rb

@@ -0,0 +1,273 @@
+# encoding: UTF-8
+#
+# = SpaceManager.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2020 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/BTree'
+require 'perobs/EquiBlobsFile'
+require 'perobs/FlatFile'
+require 'perobs/FlatFileBlobHeader'
+
+module PEROBS
+
+  # The SpaceManager is used to keep a list of all the empty spaces in a
+  # FlatFileDB file. An empty space is described by its starting address and
+  # its length in bytes. The SpaceManager keeps a list of all the spaces and
+  # can find the best fit space when a new blob needs to be added to the
+  # FlatFileDB.
+  #
+  # The SpaceManager uses two files to store the list. The first is a file
+  # with the actual addresses. This is a set of linked address lists. Each
+  # list holds the addresses for spaces that have exactly the same size. The
+  # second file is a BTree file that serves as the index. It is used to map
+  # the length of a space to the address of the linked list for that
+  # particular length. The linked list consists of elements that only hold 2
+  # items. The actual address in the FlatFileDB and the address of the next
+  # entry in the linked list in the list file.
+  class SpaceManager
+
+    attr_reader :added_spaces, :recycled_spaces, :failed_requests
+
+    def initialize(db_dir, progressmeter, btree_order = 65)
+      @db_dir = db_dir
+      @progressmeter = progressmeter
+
+      @index = BTree.new(@db_dir, 'space_index', btree_order, @progressmeter)
+      # The space list contains blobs that have each 2 entries. The address of
+      # the space in the FlatFile and the address of the next blob in the
+      # space list file that is an entry for the same space size. An address
+      # of 0 marks the end of the list.
+      @list = EquiBlobsFile.new(@db_dir, 'space_list', @progressmeter, 2 * 8, 1)
+    end
+
+    def open
+      @index.open
+      @list.open
+      reset_stats
+    end
+
+    def close
+      if @index.is_open?
+        PEROBS.log.info "SpaceManager has currently #{@list.total_entries} " +
+          "used blobs and #{@list.total_spaces} unused blobs in list " +
+          "EquiBlobsFile"
+        PEROBS.log.info "#{@added_spaces} were added, #{@recycled_spaces} " +
+          "spaces were recycled and #{@failed_requests} requests failed"
+
+        @list.close
+        @index.close
+      end
+    end
+
+    def is_open?
+      @index.is_open?
+    end
+
+    def sync
+      @list.sync
+      @index.sync
+    end
+
+    def add_space(address, length)
+      if (list_entry_addr = @index.get(length))
+        # There is already at least one move entry for this length.
+        new_list_entry_addr = insert_space_in_list(address, list_entry_addr)
+      else
+        new_list_entry_addr = insert_space_in_list(address, 0)
+      end
+      @index.insert(length, new_list_entry_addr)
+      @added_spaces += 1
+    end
+
+    def has_space?(address, length)
+      if (list_entry_addr = @index.get(length))
+        while list_entry_addr > 0
+          blob = @list.retrieve_blob(list_entry_addr)
+          space_address, next_entry_addr = blob.unpack('QQ')
+          return true if space_address == address
+          list_entry_addr = next_entry_addr
+        end
+      end
+
+      false
+    end
+
+    def get_space(length)
+      # We use a simple exact fit strategy. All attempts to use a more
+      # elaborate scheme were actually less efficient. Non-exact matches
+      # generate new spaces for the remainder and fragment the blob file with
+      # lots of unusable small spaces. Most applications seem to have
+      # clustered their blob sizes around a number of popular sizes. So exact
+      # match is very efficient to implement and results in the highest
+      # probability that a space will be reused soon.
+      list_entry_addr = @index.get(length)
+
+      if list_entry_addr
+        blob = @list.retrieve_blob(list_entry_addr)
+        space_address, next_entry_addr = blob.unpack('QQ')
+        @list.delete_blob(list_entry_addr)
+
+        if next_entry_addr > 0
+          # Update the index entry for the length to point to the
+          # following space list entry.
+          @index.insert(length, next_entry_addr)
+        else
+          # The space list for this length is empty. Remove the entry
+          # from the index.
+          @index.remove(length)
+        end
+        @recycled_spaces += 1
+
+        # We return the length to remain compatible with the old SpaceTree
+        # API.
+        return [ space_address, length ]
+      end
+
+      @failed_requests += 1
+      nil
+    end
+
+    def clear
+      @list.clear
+      @index.clear
+      reset_stats
+    end
+
+    def erase
+      @list.erase
+      @index.erase
+    end
+
+    def check(flat_file = nil)
+      sync
+      return false unless @index.check
+      return false unless @list.check
+
+      smallest_space = nil
+      largest_space = nil
+      total_space_bytes = 0
+      space_distribution = ::Hash.new(0)
+
+      @index.each do |length, list_entry_addr|
+        if list_entry_addr <= 0
+          PEROBS.log.error "list_entry_addr (#{list_entry_addr}) " +
+            "must be positive"
+          return false
+        end
+
+        # Detect smallest and largest space
+        if smallest_space.nil? || length < smallest_space
+          smallest_space = length
+        end
+        if largest_space.nil? || length > largest_space
+          largest_space = length
+        end
+
+        known_addresses = [ list_entry_addr ]
+        entries = 0
+        while list_entry_addr > 0
+          entries += 1
+          unless (blob = @list.retrieve_blob(list_entry_addr))
+            PEROBS.log.error "SpaceManager points to non-existing " +
+              "space list entry at address #{list_entry_addr}"
+            return false
+          end
+          space_address, next_entry_addr = blob.unpack('QQ')
+
+          if known_addresses.include?(next_entry_addr)
+            PEROBS.log.error "Space list is cyclic: "
+              "#{known_addresses + next_entry_addr}"
+            return false
+          end
+          if flat_file &&
+              !flat_file.has_space?(space_address, length)
+            PEROBS.log.error "SpaceManager has space at offset " +
+              "#{space_address} of size #{length} that isn't " +
+              "available in the FlatFile."
+            return false
+          end
+          list_entry_addr = next_entry_addr
+        end
+
+        total_space_bytes += length * entries
+        space_distribution[msb(length)] += entries
+      end
+
+      PEROBS.log.info "SpaceManager stats: smallest: #{smallest_space}; " +
+        "largest: #{largest_space}; total bytes: #{total_space_bytes}; " +
+        "distribution: " +
+        "#{space_distribution.map { |l, c| "#{2 ** (l - 1)}-#{2 ** l - 1}:#{c}; " }}"
+
+      true
+    end
+
+    def to_a
+      a = []
+
+      @index.each do |length, list_entry_addr|
+        while list_entry_addr > 0
+          blob = @list.retrieve_blob(list_entry_addr)
+          space_address, next_entry_addr = blob.unpack('QQ')
+
+          a << [ space_address, length ]
+
+          list_entry_addr = next_entry_addr
+        end
+      end
+
+      a.sort { |a, b| a[0] <=> b[0] }
+    end
+
+    private
+
+    def insert_space_in_list(next_element_addr, space_address)
+      blob = [ next_element_addr, space_address ].pack('QQ')
+      @list.store_blob(blob_addr = @list.free_address, blob)
+
+      blob_addr
+    end
+
+    def msb(i)
+      return 63 if i < 0
+
+      bit = 0
+      while (i > 0)
+        bit += 1
+        i = i >> 1
+      end
+
+      bit
+    end
+
+    def reset_stats
+      @added_spaces = 0
+      @recycled_spaces = 0
+      @failed_requests = 0
+    end
+
+  end
+
+end
+

data/lib/perobs/SpaceTree.rb

@@ -2,7 +2,7 @@
 #
 # = SpaceTree.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2016, 2017, 2018 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -40,20 +40,21 @@ module PEROBS
   # size which drastically simplifies the backing store operation.
   class SpaceTree
 
-    attr_reader :nodes, :cache
+    attr_reader :nodes, :cache, :progressmeter
 
     # Manage the free spaces tree in the specified directory
     # @param dir [String] directory path of an existing directory
-    def initialize(dir)
+    def initialize(dir, progressmeter)
       @dir = dir
+      @progressmeter = progressmeter
 
       # This EquiBlobsFile contains the nodes of the SpaceTree.
-      @nodes = EquiBlobsFile.new(@dir, 'database_spaces',
+      @nodes = EquiBlobsFile.new(@dir, 'database_spaces', progressmeter,
                                  SpaceTreeNode::NODE_BYTES, 1)
 
       # Benchmark runs showed a cache size of 128 to be a good compromise
       # between read and write performance trade-offs and memory consumption.
-      @cache = PersistentObjectCache.new(128, SpaceTreeNode, self)
+      @cache = PersistentObjectCache.new(256, 256, SpaceTreeNode, self)
     end
 
     # Open the SpaceTree file.
@@ -74,9 +75,14 @@ module PEROBS
       @cache.clear
     end
 
+    # @return true if file is currently open.
+    def is_open?
+      !@root_address.nil?
+    end
+
     # Flush all pending writes to the file system.
     def sync
-      @cache.flush
+      @cache.flush(true)
       @nodes.sync
     end
 
@@ -105,10 +111,12 @@ module PEROBS
       if size <= 0
         PEROBS.log.fatal "Size (#{size}) must be larger than 0."
       end
-      if has_space?(address, size)
-        PEROBS.log.fatal "The space with address #{address} and size #{size} " +
-          "can't be added twice."
-      end
+      # The following check is fairly costly and should never trigger unless
+      # there is a bug in the PEROBS code. Only use this for debugging.
+      #if has_space?(address, size)
+      #  PEROBS.log.fatal "The space with address #{address} and size " +
+      #    "#{size} can't be added twice."
+      #end
       root.add_space(address, size)
     end
 
@@ -157,8 +165,9 @@ module PEROBS
     # @param flat_file [FlatFile] Flat file to compare with
     # @return True if space list matches, flase otherwise
     def check(flat_file = nil)
-      @nodes.check
-      root.check(flat_file)
+      sync
+      return false unless @nodes.check
+      root.check(flat_file, @nodes.total_entries)
     end
 
     # Iterate over all entries and yield address and size.

data/lib/perobs/SpaceTreeNode.rb

@@ -74,24 +74,11 @@ module PEROBS
       @smaller = smaller
       @equal = equal
       @larger = larger
-
-      ObjectSpace.define_finalizer(
-        self, SpaceTreeNode._finalize(@tree, @node_address, object_id))
-      @tree.cache.insert(self, false)
-    end
-
-    # This method generates the destructor for the objects of this class. It
-    # is done this way to prevent the Proc object hanging on to a reference to
-    # self which would prevent the object from being collected. This internal
-    # method is not intended for users to call.
-    def SpaceTreeNode._finalize(tree, node_address, ruby_object_id)
-      proc { tree.cache._collect(node_address, ruby_object_id) }
     end
 
     # Create a new SpaceTreeNode. This method should be used for the creation
     # of new nodes instead of calling the constructor directly.
     # @param tree [SpaceTree] The tree the node should belong to
-    # @param node_address [Integer] Address of the node in the file
     # @param blob_address [Integer] Address of the free space blob
     # @param size [Integer] Size of the free space blob
     # @param parent [SpaceTreeNode] Parent node in the tree
@@ -99,7 +86,7 @@ module PEROBS
       node_address = tree.nodes.free_address
 
       node = SpaceTreeNode.new(tree, node_address, blob_address, size, parent)
-      node.save
+      tree.cache.insert(node)
 
       node
     end
@@ -107,7 +94,7 @@ module PEROBS
     # Restore a node from the backing store at the given address and tree.
     # @param tree [SpaceTree] The tree the node belongs to
     # @param node_address [Integer] The address in the file.
-    def SpaceTreeNode::load(tree, node_address)
+    def SpaceTreeNode::load(tree, node_address, unused = nil)
       unless node_address > 0
         PEROBS.log.fatal "node_address (#{node_address}) must be larger than 0"
       end
@@ -132,6 +119,8 @@ module PEROBS
       node = SpaceTreeNode.new(tree, node_address, blob_address, size,
                                parent, smaller, equal, larger)
 
+      tree.cache.insert(node, false)
+
       node
     end
 
@@ -529,62 +518,65 @@ module PEROBS
     # errors.
     # @param flat_file [FlatFile] If given, check that the space is also
     #        present in the given flat file.
+    # @param count [Integer] The total number of entries in the tree
     # @return [false,true] True if OK, false otherwise
-    def check(flat_file)
+    def check(flat_file, count)
       node_counter = 0
       max_depth = 0
 
-      each do |node, mode, stack|
-        max_depth = stack.size if stack.size > max_depth
-
-        case mode
-        when :smaller
-          if node.smaller
-            return false unless node.check_node_link('smaller', stack)
-            smaller_node = node.smaller
-            if smaller_node.size >= node.size
-              PEROBS.log.error "Smaller SpaceTreeNode size " +
-                "(#{smaller_node}) is not smaller than #{node}"
-              return false
+      @tree.progressmeter.start('Checking space list entries', count) do |pm|
+        each do |node, mode, stack|
+          max_depth = stack.size if stack.size > max_depth
+
+          case mode
+          when :smaller
+            if node.smaller
+              return false unless node.check_node_link('smaller', stack)
+              smaller_node = node.smaller
+              if smaller_node.size >= node.size
+                PEROBS.log.error "Smaller SpaceTreeNode size " +
+                  "(#{smaller_node}) is not smaller than #{node}"
+                return false
+              end
             end
-          end
-        when :equal
-          if node.equal
-            return false unless node.check_node_link('equal', stack)
-            equal_node = node.equal
-
-            if equal_node.smaller || equal_node.larger
-              PEROBS.log.error "Equal node #{equal_node} must not have " +
-                "smaller/larger childs"
-              return false
+          when :equal
+            if node.equal
+              return false unless node.check_node_link('equal', stack)
+              equal_node = node.equal
+
+              if equal_node.smaller || equal_node.larger
+                PEROBS.log.error "Equal node #{equal_node} must not have " +
+                  "smaller/larger childs"
+                return false
+              end
+
+              if node.size != equal_node.size
+                PEROBS.log.error "Equal SpaceTreeNode size (#{equal_node}) " +
+                  "is not equal parent node #{node}"
+                  return false
+              end
             end
-
-            if node.size != equal_node.size
-              PEROBS.log.error "Equal SpaceTreeNode size (#{equal_node}) is " +
-                "not equal parent node #{node}"
-              return false
+          when :larger
+            if node.larger
+              return false unless node.check_node_link('larger', stack)
+              larger_node = node.larger
+              if larger_node.size <= node.size
+                PEROBS.log.error "Larger SpaceTreeNode size " +
+                  "(#{larger_node}) is not larger than #{node}"
+                return false
+              end
             end
-          end
-        when :larger
-          if node.larger
-            return false unless node.check_node_link('larger', stack)
-            larger_node = node.larger
-            if larger_node.size <= node.size
-              PEROBS.log.error "Larger SpaceTreeNode size " +
-                "(#{larger_node}) is not larger than #{node}"
+          when :on_exit
+            if flat_file &&
+              !flat_file.has_space?(node.blob_address, node.size)
+              PEROBS.log.error "SpaceTreeNode has space at offset " +
+                "#{node.blob_address} of size #{node.size} that isn't " +
+                "available in the FlatFile."
               return false
             end
-          end
-        when :on_exit
-          if flat_file &&
-             !flat_file.has_space?(node.blob_address, node.size)
-            PEROBS.log.error "SpaceTreeNode has space at offset " +
-              "#{node.blob_address} of size #{node.size} that isn't " +
-              "available in the FlatFile."
-            return false
-          end
 
-          node_counter += 1
+            pm.update(node_counter += 1)
+          end
         end
       end
       PEROBS.log.debug "#{node_counter} SpaceTree nodes checked"