perobs 4.0.0 → 4.1.0

data/lib/perobs/SpaceTree.rb

@@ -2,7 +2,7 @@
 #
 # = SpaceTree.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2016, 2017, 2018 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -40,20 +40,21 @@ module PEROBS
   # size which drastically simplifies the backing store operation.
   class SpaceTree
 
-    attr_reader :nodes, :cache
+    attr_reader :nodes, :cache, :progressmeter
 
     # Manage the free spaces tree in the specified directory
     # @param dir [String] directory path of an existing directory
-    def initialize(dir)
+    def initialize(dir, progressmeter)
       @dir = dir
+      @progressmeter = progressmeter
 
       # This EquiBlobsFile contains the nodes of the SpaceTree.
-      @nodes = EquiBlobsFile.new(@dir, 'database_spaces',
+      @nodes = EquiBlobsFile.new(@dir, 'database_spaces', progressmeter,
                                  SpaceTreeNode::NODE_BYTES, 1)
 
       # Benchmark runs showed a cache size of 128 to be a good compromise
       # between read and write performance trade-offs and memory consumption.
-      @cache = PersistentObjectCache.new(128, SpaceTreeNode, self)
+      @cache = PersistentObjectCache.new(128, 5000, SpaceTreeNode, self)
     end
 
     # Open the SpaceTree file.
@@ -74,9 +75,14 @@ module PEROBS
       @cache.clear
     end
 
+    # @return true if file is currently open.
+    def is_open?
+      !@root_address.nil?
+    end
+
     # Flush all pending writes to the file system.
     def sync
-      @cache.flush
+      @cache.flush(true)
       @nodes.sync
     end
 
@@ -105,10 +111,12 @@ module PEROBS
       if size <= 0
         PEROBS.log.fatal "Size (#{size}) must be larger than 0."
       end
-      if has_space?(address, size)
-        PEROBS.log.fatal "The space with address #{address} and size #{size} " +
-          "can't be added twice."
-      end
+      # The following check is fairly costly and should never trigger unless
+      # there is a bug in the PEROBS code. Only use this for debugging.
+      #if has_space?(address, size)
+      #  PEROBS.log.fatal "The space with address #{address} and size " +
+      #    "#{size} can't be added twice."
+      #end
       root.add_space(address, size)
     end
 
@@ -157,8 +165,9 @@ module PEROBS
     # @param flat_file [FlatFile] Flat file to compare with
     # @return True if space list matches, flase otherwise
     def check(flat_file = nil)
-      @nodes.check
-      root.check(flat_file)
+      sync
+      return false unless @nodes.check
+      root.check(flat_file, @nodes.total_entries)
     end
 
     # Iterate over all entries and yield address and size.

data/lib/perobs/SpaceTreeNode.rb

@@ -74,24 +74,11 @@ module PEROBS
       @smaller = smaller
       @equal = equal
       @larger = larger
-
-      ObjectSpace.define_finalizer(
-        self, SpaceTreeNode._finalize(@tree, @node_address, object_id))
-      @tree.cache.insert(self, false)
-    end
-
-    # This method generates the destructor for the objects of this class. It
-    # is done this way to prevent the Proc object hanging on to a reference to
-    # self which would prevent the object from being collected. This internal
-    # method is not intended for users to call.
-    def SpaceTreeNode._finalize(tree, node_address, ruby_object_id)
-      proc { tree.cache._collect(node_address, ruby_object_id) }
     end
 
     # Create a new SpaceTreeNode. This method should be used for the creation
     # of new nodes instead of calling the constructor directly.
     # @param tree [SpaceTree] The tree the node should belong to
-    # @param node_address [Integer] Address of the node in the file
     # @param blob_address [Integer] Address of the free space blob
     # @param size [Integer] Size of the free space blob
     # @param parent [SpaceTreeNode] Parent node in the tree
@@ -99,7 +86,7 @@ module PEROBS
       node_address = tree.nodes.free_address
 
       node = SpaceTreeNode.new(tree, node_address, blob_address, size, parent)
-      node.save
+      tree.cache.insert(node)
 
       node
     end
@@ -107,7 +94,7 @@ module PEROBS
     # Restore a node from the backing store at the given address and tree.
     # @param tree [SpaceTree] The tree the node belongs to
     # @param node_address [Integer] The address in the file.
-    def SpaceTreeNode::load(tree, node_address)
+    def SpaceTreeNode::load(tree, node_address, unused = nil)
       unless node_address > 0
         PEROBS.log.fatal "node_address (#{node_address}) must be larger than 0"
       end
@@ -132,6 +119,8 @@ module PEROBS
       node = SpaceTreeNode.new(tree, node_address, blob_address, size,
                                parent, smaller, equal, larger)
 
+      tree.cache.insert(node, false)
+
       node
     end
 
@@ -529,62 +518,65 @@ module PEROBS
     # errors.
     # @param flat_file [FlatFile] If given, check that the space is also
     #        present in the given flat file.
+    # @param count [Integer] The total number of entries in the tree
     # @return [false,true] True if OK, false otherwise
-    def check(flat_file)
+    def check(flat_file, count)
       node_counter = 0
       max_depth = 0
 
-      each do |node, mode, stack|
-        max_depth = stack.size if stack.size > max_depth
-
-        case mode
-        when :smaller
-          if node.smaller
-            return false unless node.check_node_link('smaller', stack)
-            smaller_node = node.smaller
-            if smaller_node.size >= node.size
-              PEROBS.log.error "Smaller SpaceTreeNode size " +
-                "(#{smaller_node}) is not smaller than #{node}"
-              return false
+      @tree.progressmeter.start('Checking space list entries', count) do |pm|
+        each do |node, mode, stack|
+          max_depth = stack.size if stack.size > max_depth
+
+          case mode
+          when :smaller
+            if node.smaller
+              return false unless node.check_node_link('smaller', stack)
+              smaller_node = node.smaller
+              if smaller_node.size >= node.size
+                PEROBS.log.error "Smaller SpaceTreeNode size " +
+                  "(#{smaller_node}) is not smaller than #{node}"
+                return false
+              end
             end
-          end
-        when :equal
-          if node.equal
-            return false unless node.check_node_link('equal', stack)
-            equal_node = node.equal
-
-            if equal_node.smaller || equal_node.larger
-              PEROBS.log.error "Equal node #{equal_node} must not have " +
-                "smaller/larger childs"
-              return false
+          when :equal
+            if node.equal
+              return false unless node.check_node_link('equal', stack)
+              equal_node = node.equal
+
+              if equal_node.smaller || equal_node.larger
+                PEROBS.log.error "Equal node #{equal_node} must not have " +
+                  "smaller/larger childs"
+                return false
+              end
+
+              if node.size != equal_node.size
+                PEROBS.log.error "Equal SpaceTreeNode size (#{equal_node}) " +
+                  "is not equal parent node #{node}"
+                  return false
+              end
             end
-
-            if node.size != equal_node.size
-              PEROBS.log.error "Equal SpaceTreeNode size (#{equal_node}) is " +
-                "not equal parent node #{node}"
-              return false
+          when :larger
+            if node.larger
+              return false unless node.check_node_link('larger', stack)
+              larger_node = node.larger
+              if larger_node.size <= node.size
+                PEROBS.log.error "Larger SpaceTreeNode size " +
+                  "(#{larger_node}) is not larger than #{node}"
+                return false
+              end
             end
-          end
-        when :larger
-          if node.larger
-            return false unless node.check_node_link('larger', stack)
-            larger_node = node.larger
-            if larger_node.size <= node.size
-              PEROBS.log.error "Larger SpaceTreeNode size " +
-                "(#{larger_node}) is not larger than #{node}"
+          when :on_exit
+            if flat_file &&
+              !flat_file.has_space?(node.blob_address, node.size)
+              PEROBS.log.error "SpaceTreeNode has space at offset " +
+                "#{node.blob_address} of size #{node.size} that isn't " +
+                "available in the FlatFile."
               return false
             end
-          end
-        when :on_exit
-          if flat_file &&
-             !flat_file.has_space?(node.blob_address, node.size)
-            PEROBS.log.error "SpaceTreeNode has space at offset " +
-              "#{node.blob_address} of size #{node.size} that isn't " +
-              "available in the FlatFile."
-            return false
-          end
 
-          node_counter += 1
+            pm.update(node_counter += 1)
+          end
         end
       end
       PEROBS.log.debug "#{node_counter} SpaceTree nodes checked"

data/lib/perobs/Store.rb

@@ -2,7 +2,8 @@
 #
 # = Store.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2017, 2018, 2019
+# by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -36,6 +37,11 @@ require 'perobs/FlatFileDB'
 require 'perobs/Object'
 require 'perobs/Hash'
 require 'perobs/Array'
+require 'perobs/BigTree'
+require 'perobs/BigHash'
+require 'perobs/BigArray'
+require 'perobs/ProgressMeter'
+require 'perobs/ConsoleProgressMeter'
 
 # PErsistent Ruby OBject Store
 module PEROBS
@@ -132,8 +138,14 @@ module PEROBS
     #                      :yaml : Can also handle most Ruby data types and is
     #                      portable between Ruby versions (1.9 and later).
     #                      Unfortunately, it is 10x slower than marshal.
+    #     :progressmeter : reference to a ProgressMeter object that receives
+    #                      progress information during longer running tasks.
+    #                      It defaults to ProgressMeter which only logs into
+    #                      the log. Use ConsoleProgressMeter or a derived
+    #                      class for more fancy progress reporting.
     def initialize(data_base, options = {})
       # Create a backing store handler
+      @progressmeter = (options[:progressmeter] ||= ProgressMeter.new)
       @db = (options[:engine] || FlatFileDB).new(data_base, options)
       @db.open
       # Create a map that can translate classes to numerical IDs and vice
@@ -197,7 +209,10 @@ module PEROBS
     # method was called.
     def exit
       if @cache && @cache.in_transaction?
-        PEROBS.log.fatal 'You cannot call exit() during a transaction'
+        @cache.abort_transaction
+        @cache.flush
+        @db.close if @db
+        PEROBS.log.fatal "You cannot call exit() during a transaction: #{Kernel.caller}"
       end
       @cache.flush if @cache
       @db.close if @db
@@ -297,20 +312,30 @@ module PEROBS
     # needed.
     def sync
       if @cache.in_transaction?
-        PEROBS.log.fatal 'You cannot call sync() during a transaction'
+        @cache.abort_transaction
+        @cache.flush
+        PEROBS.log.fatal "You cannot call sync() during a transaction: \n" +
+          Kernel.caller.join("\n")
       end
       @cache.flush
     end
 
+    # Return the number of object stored in the store. CAVEAT: This method
+    # will only return correct values when it is separated from any mutating
+    # call by a call to sync().
+    # @return [Integer] Number of persistently stored objects in the Store.
+    def size
+      # We don't include the Hash that stores the root objects into the object
+      # count.
+      @db.item_counter - 1
+    end
+
     # Discard all objects that are not somehow connected to the root objects
     # from the back-end storage. The garbage collector is not invoked
     # automatically. Depending on your usage pattern, you need to call this
     # method periodically.
     # @return [Integer] The number of collected objects
     def gc
-      if @cache.in_transaction?
-        PEROBS.log.fatal 'You cannot call gc() during a transaction'
-      end
       sync
       mark
       sweep
@@ -354,10 +379,6 @@ module PEROBS
         return obj
       end
 
-      #if (obj = @db.search_object(id))
-      #  PEROBS.log.fatal "Object was not in index but in DB"
-      #end
-
       # The requested object does not exist. Return nil.
       nil
     end
@@ -370,38 +391,42 @@ module PEROBS
     #        made.
     # @return [Integer] The number of references to bad objects found.
     def check(repair = false)
+      stats = { :errors => 0, :object_cnt => 0 }
+
       # All objects must have in-db version.
       sync
       # Run basic consistency checks first.
-      errors = @db.check_db(repair)
+      stats[:errors] += @db.check_db(repair)
 
       # We will use the mark to mark all objects that we have checked already.
       # Before we start, we need to clear all marks.
       @db.clear_marks
 
-      objects = 0
-      @root_objects.each do |name, id|
-        objects += 1
-        errors += check_object(id, repair)
+      @progressmeter.start("Checking object link structure",
+                           @db.item_counter) do
+        @root_objects.each do |name, id|
+          check_object(id, repair, stats)
+        end
       end
 
       # Delete all broken root objects.
       if repair
         @root_objects.delete_if do |name, id|
-          unless (res = @db.check(id, repair))
+          unless @db.check(id, repair)
             PEROBS.log.error "Discarding broken root object '#{name}' " +
               "with ID #{id}"
-            errors += 1
+            stats[:errors] += 1
           end
-          !res
         end
       end
 
-      if errors > 0
+      if stats[:errors] > 0
         if repair
-          PEROBS.log.error "#{errors} errors found in #{objects} objects"
+          PEROBS.log.error "#{stats[:errors]} errors found in " +
+            "#{stats[:object_cnt]} objects"
         else
-          PEROBS.log.fatal "#{errors} errors found in #{objects} objects"
+          PEROBS.log.fatal "#{stats[:errors]} errors found in " +
+            "#{stats[:object_cnt]} objects"
         end
       else
         PEROBS.log.debug "No errors found"
@@ -410,7 +435,7 @@ module PEROBS
       # Ensure that any fixes are written into the DB.
       sync if repair
 
-      errors
+      stats[:errors]
     end
 
     # This method will execute the provided block as an atomic transaction
@@ -439,7 +464,10 @@ module PEROBS
       stack = [ 0 ] + @root_objects.values
       while !stack.empty?
         # Get an object index from the stack.
-        unless (obj = object_by_id(id = stack.pop))
+        id = stack.pop
+        next if @db.is_marked?(id)
+
+        unless (obj = object_by_id(id))
           PEROBS.log.fatal "Database is corrupted. Object with ID #{id} " +
             "not found."
         end
@@ -510,19 +538,25 @@ module PEROBS
     def mark
       classes = Set.new
       marked_objects = 0
-      each { |obj| classes.add(obj.class); marked_objects += 1 }
+      @progressmeter.start("Marking linked objects", @db.item_counter) do
+        each do |obj|
+          classes.add(obj.class)
+          @progressmeter.update(marked_objects += 1)
+        end
+      end
       @class_map.keep(classes.map { |c| c.to_s })
 
       # The root_objects object is included in the count, but we only want to
       # count user objects here.
-      PEROBS.log.debug "#{marked_objects - 1} objects marked"
+      PEROBS.log.debug "#{marked_objects - 1} of #{@db.item_counter} " +
+        "objects marked"
       @stats.marked_objects = marked_objects - 1
     end
 
     # Sweep phase of a mark-and-sweep garbage collector. It will remove all
     # unmarked objects from the store.
     def sweep
-      @stats.swept_objects = @db.delete_unmarked_objects.length
+      @stats.swept_objects = @db.delete_unmarked_objects
       @cache.reset
       PEROBS.log.debug "#{@stats.swept_objects} objects collected"
       @stats.swept_objects
@@ -534,8 +568,7 @@ module PEROBS
     #        with
     # @param repair [Boolean] Delete refernces to broken objects if true
     # @return [Integer] The number of references to bad objects.
-    def check_object(start_id, repair)
-      errors = 0
+    def check_object(start_id, repair, stats)
       @db.mark(start_id)
       # The todo list holds a touple for each object that still needs to be
       # checked. The first item is the referring object and the second is the
@@ -546,7 +579,13 @@ module PEROBS
         # Get the next PEROBS object to check
         ref_obj, id = todo_list.pop
 
-        if (obj = object_by_id(id))
+        begin
+          obj = object_by_id(id)
+        rescue PEROBS::FatalError
+          obj = nil
+        end
+
+        if obj
           # The object exists and is OK. Mark is as checked.
           @db.mark(id)
           # Now look at all other objects referenced by this object.
@@ -569,11 +608,11 @@ module PEROBS
                 ref_obj.inspect
             end
           end
-          errors += 1
+          stats[:errors] += 1
         end
-      end
 
-      errors
+        @progressmeter.update(stats[:object_cnt] += 1)
+      end
     end
 
   end