perobs 3.0.1 → 4.3.0

data/lib/perobs/ProgressMeter.rb

@@ -0,0 +1,97 @@
+# encoding: UTF-8
+#
+# = ProgressMeter.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2018 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'time'
+
+module PEROBS
+
+  # This is the base class for all ProgressMeter classes. It only logs into
+  # the PEROBS log. You need to create a derived class that overloads
+  # print_bar() and print_time() to provide more fancy outputs.
+  class ProgressMeter
+
+    def initialize
+      @name = nil
+      @max_value = nil
+      @current_value = nil
+      @start_time = nil
+      @end_time = nil
+    end
+
+    def start(name, max_value)
+      @name = name
+      unless max_value >= 0
+        raise ArgumentError, "Maximum value (#{max_value}) must be larger " +
+          "or equal to 0"
+      end
+      @max_value = max_value
+      @current_value = 0
+      @start_time = Time.now
+      @end_time = nil
+      print_bar
+
+      if block_given?
+        yield(self)
+        done
+      end
+    end
+
+    def update(value)
+      return unless (value_i = value.to_i) > @current_value
+
+      @current_value = value_i
+      print_bar
+    end
+
+    def done
+      @end_time = Time.now
+      print_time
+      PEROBS.log.info "#{@name} completed in " +
+        secsToHMS(@end_time - @start_time)
+    end
+
+    private
+
+    def print_bar
+    end
+
+    def print_time
+    end
+
+    def secsToHMS(secs)
+      secs = secs.to_i
+      s = secs % 60
+      mins = secs / 60
+      m = mins % 60
+      h = mins / 60
+      "#{h}:#{'%02d' % m}:#{'%02d' % s}"
+    end
+
+  end
+
+end
+

data/lib/perobs/SpaceManager.rb

@@ -0,0 +1,273 @@
+# encoding: UTF-8
+#
+# = SpaceManager.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2020 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/BTree'
+require 'perobs/EquiBlobsFile'
+require 'perobs/FlatFile'
+require 'perobs/FlatFileBlobHeader'
+
+module PEROBS
+
+  # The SpaceManager is used to keep a list of all the empty spaces in a
+  # FlatFileDB file. An empty space is described by its starting address and
+  # its length in bytes. The SpaceManager keeps a list of all the spaces and
+  # can find the best fit space when a new blob needs to be added to the
+  # FlatFileDB.
+  #
+  # The SpaceManager uses two files to store the list. The first is a file
+  # with the actual addresses. This is a set of linked address lists. Each
+  # list holds the addresses for spaces that have exactly the same size. The
+  # second file is a BTree file that serves as the index. It is used to map
+  # the length of a space to the address of the linked list for that
+  # particular length. The linked list consists of elements that only hold 2
+  # items. The actual address in the FlatFileDB and the address of the next
+  # entry in the linked list in the list file.
+  class SpaceManager
+
+    attr_reader :added_spaces, :recycled_spaces, :failed_requests
+
+    def initialize(db_dir, progressmeter, btree_order = 65)
+      @db_dir = db_dir
+      @progressmeter = progressmeter
+
+      @index = BTree.new(@db_dir, 'space_index', btree_order, @progressmeter)
+      # The space list contains blobs that have each 2 entries. The address of
+      # the space in the FlatFile and the address of the next blob in the
+      # space list file that is an entry for the same space size. An address
+      # of 0 marks the end of the list.
+      @list = EquiBlobsFile.new(@db_dir, 'space_list', @progressmeter, 2 * 8, 1)
+    end
+
+    def open
+      @index.open
+      @list.open
+      reset_stats
+    end
+
+    def close
+      if @index.is_open?
+        PEROBS.log.info "SpaceManager has currently #{@list.total_entries} " +
+          "used blobs and #{@list.total_spaces} unused blobs in list " +
+          "EquiBlobsFile"
+        PEROBS.log.info "#{@added_spaces} were added, #{@recycled_spaces} " +
+          "spaces were recycled and #{@failed_requests} requests failed"
+
+        @list.close
+        @index.close
+      end
+    end
+
+    def is_open?
+      @index.is_open?
+    end
+
+    def sync
+      @list.sync
+      @index.sync
+    end
+
+    def add_space(address, length)
+      if (list_entry_addr = @index.get(length))
+        # There is already at least one move entry for this length.
+        new_list_entry_addr = insert_space_in_list(address, list_entry_addr)
+      else
+        new_list_entry_addr = insert_space_in_list(address, 0)
+      end
+      @index.insert(length, new_list_entry_addr)
+      @added_spaces += 1
+    end
+
+    def has_space?(address, length)
+      if (list_entry_addr = @index.get(length))
+        while list_entry_addr > 0
+          blob = @list.retrieve_blob(list_entry_addr)
+          space_address, next_entry_addr = blob.unpack('QQ')
+          return true if space_address == address
+          list_entry_addr = next_entry_addr
+        end
+      end
+
+      false
+    end
+
+    def get_space(length)
+      # We use a simple exact fit strategy. All attempts to use a more
+      # elaborate scheme were actually less efficient. Non-exact matches
+      # generate new spaces for the remainder and fragment the blob file with
+      # lots of unusable small spaces. Most applications seem to have
+      # clustered their blob sizes around a number of popular sizes. So exact
+      # match is very efficient to implement and results in the highest
+      # probability that a space will be reused soon.
+      list_entry_addr = @index.get(length)
+
+      if list_entry_addr
+        blob = @list.retrieve_blob(list_entry_addr)
+        space_address, next_entry_addr = blob.unpack('QQ')
+        @list.delete_blob(list_entry_addr)
+
+        if next_entry_addr > 0
+          # Update the index entry for the length to point to the
+          # following space list entry.
+          @index.insert(length, next_entry_addr)
+        else
+          # The space list for this length is empty. Remove the entry
+          # from the index.
+          @index.remove(length)
+        end
+        @recycled_spaces += 1
+
+        # We return the length to remain compatible with the old SpaceTree
+        # API.
+        return [ space_address, length ]
+      end
+
+      @failed_requests += 1
+      nil
+    end
+
+    def clear
+      @list.clear
+      @index.clear
+      reset_stats
+    end
+
+    def erase
+      @list.erase
+      @index.erase
+    end
+
+    def check(flat_file = nil)
+      sync
+      return false unless @index.check
+      return false unless @list.check
+
+      smallest_space = nil
+      largest_space = nil
+      total_space_bytes = 0
+      space_distribution = ::Hash.new(0)
+
+      @index.each do |length, list_entry_addr|
+        if list_entry_addr <= 0
+          PEROBS.log.error "list_entry_addr (#{list_entry_addr}) " +
+            "must be positive"
+          return false
+        end
+
+        # Detect smallest and largest space
+        if smallest_space.nil? || length < smallest_space
+          smallest_space = length
+        end
+        if largest_space.nil? || length > largest_space
+          largest_space = length
+        end
+
+        known_addresses = [ list_entry_addr ]
+        entries = 0
+        while list_entry_addr > 0
+          entries += 1
+          unless (blob = @list.retrieve_blob(list_entry_addr))
+            PEROBS.log.error "SpaceManager points to non-existing " +
+              "space list entry at address #{list_entry_addr}"
+            return false
+          end
+          space_address, next_entry_addr = blob.unpack('QQ')
+
+          if known_addresses.include?(next_entry_addr)
+            PEROBS.log.error "Space list is cyclic: "
+              "#{known_addresses + next_entry_addr}"
+            return false
+          end
+          if flat_file &&
+              !flat_file.has_space?(space_address, length)
+            PEROBS.log.error "SpaceManager has space at offset " +
+              "#{space_address} of size #{length} that isn't " +
+              "available in the FlatFile."
+            return false
+          end
+          list_entry_addr = next_entry_addr
+        end
+
+        total_space_bytes += length * entries
+        space_distribution[msb(length)] += entries
+      end
+
+      PEROBS.log.info "SpaceManager stats: smallest: #{smallest_space}; " +
+        "largest: #{largest_space}; total bytes: #{total_space_bytes}; " +
+        "distribution: " +
+        "#{space_distribution.map { |l, c| "#{2 ** (l - 1)}-#{2 ** l - 1}:#{c}; " }}"
+
+      true
+    end
+
+    def to_a
+      a = []
+
+      @index.each do |length, list_entry_addr|
+        while list_entry_addr > 0
+          blob = @list.retrieve_blob(list_entry_addr)
+          space_address, next_entry_addr = blob.unpack('QQ')
+
+          a << [ space_address, length ]
+
+          list_entry_addr = next_entry_addr
+        end
+      end
+
+      a.sort { |a, b| a[0] <=> b[0] }
+    end
+
+    private
+
+    def insert_space_in_list(next_element_addr, space_address)
+      blob = [ next_element_addr, space_address ].pack('QQ')
+      @list.store_blob(blob_addr = @list.free_address, blob)
+
+      blob_addr
+    end
+
+    def msb(i)
+      return 63 if i < 0
+
+      bit = 0
+      while (i > 0)
+        bit += 1
+        i = i >> 1
+      end
+
+      bit
+    end
+
+    def reset_stats
+      @added_spaces = 0
+      @recycled_spaces = 0
+      @failed_requests = 0
+    end
+
+  end
+
+end
+

data/lib/perobs/SpaceTree.rb

@@ -2,7 +2,7 @@
 #
 # = SpaceTree.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2016, 2017, 2018 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -27,7 +27,7 @@
 
 require 'perobs/Log'
 require 'perobs/EquiBlobsFile'
-require 'perobs/SpaceTreeNodeCache'
+require 'perobs/PersistentObjectCache'
 require 'perobs/SpaceTreeNode'
 require 'perobs/FlatFile'
 
@@ -40,40 +40,63 @@ module PEROBS
   # size which drastically simplifies the backing store operation.
   class SpaceTree
 
-    attr_reader :nodes
+    attr_reader :nodes, :cache, :progressmeter
 
     # Manage the free spaces tree in the specified directory
     # @param dir [String] directory path of an existing directory
-    def initialize(dir)
+    def initialize(dir, progressmeter)
       @dir = dir
+      @progressmeter = progressmeter
 
       # This EquiBlobsFile contains the nodes of the SpaceTree.
-      @nodes = EquiBlobsFile.new(@dir, 'database_spaces',
+      @nodes = EquiBlobsFile.new(@dir, 'database_spaces', progressmeter,
                                  SpaceTreeNode::NODE_BYTES, 1)
 
-      @node_cache = SpaceTreeNodeCache.new(128)
+      # Benchmark runs showed a cache size of 128 to be a good compromise
+      # between read and write performance trade-offs and memory consumption.
+      @cache = PersistentObjectCache.new(256, 256, SpaceTreeNode, self)
     end
 
     # Open the SpaceTree file.
     def open
       @nodes.open
-      @node_cache.clear
-      @root = SpaceTreeNode.new(self, nil, @nodes.total_entries == 0 ?
-                                           nil : @nodes.first_entry)
-      @node_cache.insert(@root)
+      @cache.clear
+      node = @nodes.total_entries == 0 ?
+        SpaceTreeNode::create(self) :
+        SpaceTreeNode::load(self, @nodes.first_entry)
+      @root_address = node.node_address
     end
 
     # Close the SpaceTree file.
     def close
+      @cache.flush(true)
       @nodes.close
-      @root = nil
-      @node_cache.clear
+      @root_address = nil
+      @cache.clear
     end
 
+    # @return true if file is currently open.
+    def is_open?
+      !@root_address.nil?
+    end
+
+    # Flush all pending writes to the file system.
+    def sync
+      @cache.flush(true)
+      @nodes.sync
+    end
+
+    # Set a new root node for the SpaceTree
+    # @param node [SpaceTreeNode]
     def set_root(node)
-      @root = node
+      @root_address = node.node_address
+      @nodes.first_entry = node.node_address
     end
 
+    # Return the SpaceTreeNode that is the root of the SpaceTree.
+    def root
+      @root_address ? @cache.get(@root_address) : nil
+    end
 
     # Erase the SpaceTree file. This method cannot be called while the file is
     # open.
@@ -88,7 +111,13 @@ module PEROBS
       if size <= 0
         PEROBS.log.fatal "Size (#{size}) must be larger than 0."
       end
-      @root.add_space(address, size)
+      # The following check is fairly costly and should never trigger unless
+      # there is a bug in the PEROBS code. Only use this for debugging.
+      #if has_space?(address, size)
+      #  PEROBS.log.fatal "The space with address #{address} and size " +
+      #    "#{size} can't be added twice."
+      #end
+      root.add_space(address, size)
     end
 
     # Get a space that has at least the requested size.
@@ -99,10 +128,10 @@ module PEROBS
         PEROBS.log.fatal "Size (#{size}) must be larger than 0."
       end
 
-      if (address_size = @root.find_matching_space(size))
+      if (address_size = root.find_matching_space(size))
         # First we try to find an exact match.
         return address_size
-      elsif (address_size = @root.find_equal_or_larger_space(size))
+      elsif (address_size = root.find_equal_or_larger_space(size))
         return address_size
       else
         return nil
@@ -112,38 +141,15 @@ module PEROBS
     # Delete the node at the given address in the SpaceTree file.
     # @param address [Integer] address in file
     def delete_node(address)
-      @node_cache.delete(address)
+      @cache.delete(address)
       @nodes.delete_blob(address)
     end
 
     # Clear all pools and forget any registered spaces.
     def clear
       @nodes.clear
-      @node_cache.clear
-      @root = SpaceTreeNode.new(self)
-      @node_cache.insert(@root)
-    end
-
-    # Create a new SpaceTreeNode.
-    # @param parent [SpaceTreeNode] parent node
-    # @param blob_address [Integer] address of the free space
-    # @param size [Integer] size of the free space
-    def new_node(parent, blob_address, size)
-      node = SpaceTreeNode.new(self, parent, nil, blob_address, size)
-      @node_cache.insert(node)
-    end
-
-    # Return the SpaceTreeNode that matches the given node address. If a blob
-    # address and size are given, a new node is created instead of being read
-    # from the file.
-    # @param node_address [Integer] Address of the node in the SpaceTree file
-    # @return [SpaceTreeNode]
-    def get_node(node_address)
-      if (node = @node_cache.get(node_address))
-        return node
-      end
-
-      @node_cache.insert(SpaceTreeNode.new(self, nil, node_address))
+      @cache.clear
+      @root_address = SpaceTreeNode::create(self).node_address
     end
 
     # Check if there is a space in the free space lists that matches the
@@ -152,27 +158,37 @@ module PEROBS
     # @param [Integer] size Length of the space in bytes
     # @return [Boolean] True if space is found, false otherwise
     def has_space?(address, size)
-      @root.has_space?(address, size)
+      root.has_space?(address, size)
     end
 
     # Check if the index is OK and matches the flat_file data (if given).
     # @param flat_file [FlatFile] Flat file to compare with
     # @return True if space list matches, flase otherwise
     def check(flat_file = nil)
-      @nodes.check
-      @root.check(flat_file)
+      sync
+      return false unless @nodes.check
+      root.check(flat_file, @nodes.total_entries)
+    end
+
+    # Iterate over all entries and yield address and size.
+    def each
+      root.each do |node, mode, stack|
+        if mode == :on_enter
+          yield(node.blob_address, node.size)
+        end
+      end
     end
 
     # Complete internal tree data structure as textual tree.
     # @return [String]
     def to_s
-      @root.to_tree_s
+      root.to_tree_s
     end
 
     # Convert the tree into an Array of [address, size] touples.
     # @return [Array]
     def to_a
-      @root.to_a
+      root.to_a
     end
 
   end