perobs 4.0.0 → 4.4.0

data/lib/perobs/Store.rb

@@ -2,7 +2,8 @@
 #
 # = Store.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022
+# by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -26,6 +27,7 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 require 'set'
+require 'monitor'
 
 require 'perobs/Log'
 require 'perobs/Handle'
@@ -36,12 +38,18 @@ require 'perobs/FlatFileDB'
 require 'perobs/Object'
 require 'perobs/Hash'
 require 'perobs/Array'
+require 'perobs/BigTree'
+require 'perobs/BigHash'
+require 'perobs/BigArray'
+require 'perobs/ProgressMeter'
+require 'perobs/ConsoleProgressMeter'
 
 # PErsistent Ruby OBject Store
 module PEROBS
 
   Statistics = Struct.new(:in_memory_objects, :root_objects,
-                          :marked_objects, :swept_objects)
+                          :marked_objects, :swept_objects,
+                          :created_objects, :collected_objects)
 
   # PEROBS::Store is a persistent storage system for Ruby objects. Regular
   # Ruby objects are transparently stored in a back-end storage and retrieved
@@ -103,6 +111,7 @@ module PEROBS
   class Store
 
     attr_reader :db, :cache, :class_map
+    attr_writer :root_objects
 
     # Create a new Store.
     # @param data_base [String] the name of the database
@@ -132,8 +141,17 @@ module PEROBS
     #                      :yaml : Can also handle most Ruby data types and is
     #                      portable between Ruby versions (1.9 and later).
     #                      Unfortunately, it is 10x slower than marshal.
+    #     :progressmeter : reference to a ProgressMeter object that receives
+    #                      progress information during longer running tasks.
+    #                      It defaults to ProgressMeter which only logs into
+    #                      the log. Use ConsoleProgressMeter or a derived
+    #                      class for more fancy progress reporting.
+    #   :no_root_objects : Create a new store without root objects. This only
+    #                      makes sense if you want to copy the objects of
+    #                      another store into this store.
     def initialize(data_base, options = {})
       # Create a backing store handler
+      @progressmeter = (options[:progressmeter] ||= ProgressMeter.new)
       @db = (options[:engine] || FlatFileDB).new(data_base, options)
       @db.open
       # Create a map that can translate classes to numerical IDs and vice
@@ -146,22 +164,29 @@ module PEROBS
 
       # This objects keeps some counters of interest.
       @stats = Statistics.new
+      @stats[:created_objects] = 0
+      @stats[:collected_objects] = 0
 
       # The Cache reduces read and write latencies by keeping a subset of the
       # objects in memory.
       @cache = Cache.new(options[:cache_bits] || 16)
 
+      # Lock to serialize access to the Store and all stored data.
+      @lock = Monitor.new
+
       # The named (global) objects IDs hashed by their name
-      unless (@root_objects = object_by_id(0))
-        PEROBS.log.debug "Initializing the PEROBS store"
-        # The root object hash always has the object ID 0.
-        @root_objects = _construct_po(Hash, 0)
-        # Mark the root_objects object as modified.
-        @cache.cache_write(@root_objects)
-      end
-      unless @root_objects.is_a?(Hash)
-        PEROBS.log.fatal "Database corrupted: Root objects must be a Hash " +
-          "but is a #{@root_objects.class}"
+      unless options[:no_root_objects]
+        unless (@root_objects = object_by_id(0))
+          PEROBS.log.debug "Initializing the PEROBS store"
+          # The root object hash always has the object ID 0.
+          @root_objects = _construct_po(Hash, 0)
+          # Mark the root_objects object as modified.
+          @cache.cache_write(@root_objects)
+        end
+        unless @root_objects.is_a?(Hash)
+          PEROBS.log.fatal "Database corrupted: Root objects must be a Hash " +
+            "but is a #{@root_objects.class}"
+        end
       end
     end
 
@@ -173,7 +198,9 @@ module PEROBS
       sync
 
       # Create a new store with the specified directory and options.
-      new_db = Store.new(dir, options)
+      new_options = options.clone
+      new_options[:no_root_objects] = true
+      new_db = Store.new(dir, new_options)
       # Clear the cache.
       new_db.sync
       # Copy all objects of the existing store to the new store.
@@ -184,6 +211,7 @@ module PEROBS
         obj._sync
         i += 1
       end
+      new_db.root_objects = new_db.object_by_id(0)
       PEROBS.log.debug "Copied #{i} objects into new database at #{dir}"
       # Flush the new store and close it.
       new_db.exit
@@ -191,20 +219,34 @@ module PEROBS
       true
     end
 
-
     # Close the store and ensure that all in-memory objects are written out to
     # the storage backend. The Store object is no longer usable after this
     # method was called.
     def exit
       if @cache && @cache.in_transaction?
-        PEROBS.log.fatal 'You cannot call exit() during a transaction'
+        @cache.abort_transaction
+        @cache.flush
+        @db.close if @db
+        PEROBS.log.fatal "You cannot call exit() during a transaction: #{Kernel.caller}"
       end
       @cache.flush if @cache
       @db.close if @db
-      @db = @class_map = @in_memory_objects = @stats = @cache = @root_objects =
-        nil
-    end
 
+      GC.start
+      if @stats
+        unless @stats[:created_objects] == @stats[:collected_objects] +
+            @in_memory_objects.length
+          PEROGS.log.fatal "Created objects count " +
+            "(#{@stats[:created_objects]})" +
+            " is not equal to the collected count " +
+            "(#{@stats[:collected_objects]}) + in_memory_objects count " +
+            "(#{@in_memory_objects.length})"
+        end
+      end
+
+      @db = @class_map = @in_memory_objects = @stats = @cache =
+        @root_objects = nil
+    end
 
     # You need to call this method to create new PEROBS objects that belong to
     # this Store.
@@ -218,11 +260,13 @@ module PEROBS
         PEROBS.log.fatal "#{klass} is not a BasicObject derivative"
       end
 
-      obj = _construct_po(klass, _new_id, *args)
-      # Mark the new object as modified so it gets pushed into the database.
-      @cache.cache_write(obj)
-      # Return a POXReference proxy for the newly created object.
-      obj.myself
+      @lock.synchronize do
+        obj = _construct_po(klass, _new_id, *args)
+        # Mark the new object as modified so it gets pushed into the database.
+        @cache.cache_write(obj)
+        # Return a POXReference proxy for the newly created object.
+        obj.myself
+      end
     end
 
     # For library internal use only!
@@ -239,9 +283,11 @@ module PEROBS
     # method was called. This is an alternative to exit() that additionaly
     # deletes the entire database.
     def delete_store
-      @db.delete_database
-      @db = @class_map = @in_memory_objects = @stats = @cache = @root_objects =
-        nil
+      @lock.synchronize do
+        @db.delete_database
+        @db = @class_map = @in_memory_objects = @stats = @cache =
+          @root_objects = nil
+      end
     end
 
     # Store the provided object under the given name. Use this to make the
@@ -253,25 +299,27 @@ module PEROBS
     # @param obj [PEROBS::Object] The object to store
     # @return [PEROBS::Object] The stored object.
     def []=(name, obj)
-      # If the passed object is nil, we delete the entry if it exists.
-      if obj.nil?
-        @root_objects.delete(name)
-        return nil
-      end
+      @lock.synchronize do
+        # If the passed object is nil, we delete the entry if it exists.
+        if obj.nil?
+          @root_objects.delete(name)
+          return nil
+        end
 
-      # We only allow derivatives of PEROBS::Object to be stored in the
-      # store.
-      unless obj.is_a?(ObjectBase)
-        PEROBS.log.fatal 'Object must be of class PEROBS::Object but ' +
-          "is of class #{obj.class}"
-      end
+        # We only allow derivatives of PEROBS::Object to be stored in the
+        # store.
+        unless obj.is_a?(ObjectBase)
+          PEROBS.log.fatal 'Object must be of class PEROBS::Object but ' +
+            "is of class #{obj.class}"
+        end
 
-      unless obj.store == self
-        PEROBS.log.fatal 'The object does not belong to this store.'
-      end
+        unless obj.store == self
+          PEROBS.log.fatal 'The object does not belong to this store.'
+        end
 
-      # Store the name and mark the name list as modified.
-      @root_objects[name] = obj._id
+        # Store the name and mark the name list as modified.
+        @root_objects[name] = obj._id
+      end
 
       obj
     end
@@ -281,25 +329,46 @@ module PEROBS
     #        returned.
     # @return The requested object or nil if it doesn't exist.
     def [](name)
-      # Return nil if there is no object with that name.
-      return nil unless (id = @root_objects[name])
+      @lock.synchronize do
+        # Return nil if there is no object with that name.
+        return nil unless (id = @root_objects[name])
 
-      POXReference.new(self, id)
+        POXReference.new(self, id)
+      end
     end
 
     # Return a list with all the names of the root objects.
     # @return [Array of Symbols]
     def names
-      @root_objects.keys
+      @lock.synchronize do
+        @root_objects.keys
+      end
     end
 
     # Flush out all modified objects to disk and shrink the in-memory list if
     # needed.
     def sync
-      if @cache.in_transaction?
-        PEROBS.log.fatal 'You cannot call sync() during a transaction'
+      @lock.synchronize do
+        if @cache.in_transaction?
+          @cache.abort_transaction
+          @cache.flush
+          PEROBS.log.fatal "You cannot call sync() during a transaction: \n" +
+            Kernel.caller.join("\n")
+        end
+        @cache.flush
+      end
+    end
+
+    # Return the number of object stored in the store. CAVEAT: This method
+    # will only return correct values when it is separated from any mutating
+    # call by a call to sync().
+    # @return [Integer] Number of persistently stored objects in the Store.
+    def size
+      # We don't include the Hash that stores the root objects into the object
+      # count.
+      @lock.synchronize do
+        @db.item_counter - 1
       end
-      @cache.flush
     end
 
     # Discard all objects that are not somehow connected to the root objects
@@ -308,58 +377,20 @@ module PEROBS
     # method periodically.
     # @return [Integer] The number of collected objects
     def gc
-      if @cache.in_transaction?
-        PEROBS.log.fatal 'You cannot call gc() during a transaction'
+      @lock.synchronize do
+        sync
+        mark
+        sweep
       end
-      sync
-      mark
-      sweep
     end
 
     # Return the object with the provided ID. This method is not part of the
     # public API and should never be called by outside users. It's purely
     # intended for internal use.
     def object_by_id(id)
-      if (ruby_object_id = @in_memory_objects[id])
-        # We have the object in memory so we can just return it.
-        begin
-          object = ObjectSpace._id2ref(ruby_object_id)
-          # Let's make sure the object is really the object we are looking
-          # for. The GC might have recycled it already and the Ruby object ID
-          # could now be used for another object.
-          if object.is_a?(ObjectBase) && object._id == id
-            return object
-          end
-        rescue RangeError => e
-          # Due to a race condition the object can still be in the
-          # @in_memory_objects list but has been collected already by the Ruby
-          # GC. In that case we need to load it again. In this case the
-          # _collect() call will happen much later, potentially after we have
-          # registered a new object with the same ID.
-          @in_memory_objects.delete(id)
-        end
+      @lock.synchronize do
+        object_by_id_internal(id)
       end
-
-      if (obj = @cache.object_by_id(id))
-        PEROBS.log.fatal "Object #{id} with Ruby #{obj.object_id} is in cache but not in_memory"
-      end
-
-      # We don't have the object in memory. Let's find it in the storage.
-      if @db.include?(id)
-        # Great, object found. Read it into memory and return it.
-        obj = ObjectBase::read(self, id)
-        # Add the object to the in-memory storage list.
-        @cache.cache_read(obj)
-
-        return obj
-      end
-
-      #if (obj = @db.search_object(id))
-      #  PEROBS.log.fatal "Object was not in index but in DB"
-      #end
-
-      # The requested object does not exist. Return nil.
-      nil
     end
 
     # This method can be used to check the database and optionally repair it.
@@ -370,38 +401,42 @@ module PEROBS
     #        made.
     # @return [Integer] The number of references to bad objects found.
     def check(repair = false)
+      stats = { :errors => 0, :object_cnt => 0 }
+
       # All objects must have in-db version.
       sync
       # Run basic consistency checks first.
-      errors = @db.check_db(repair)
+      stats[:errors] += @db.check_db(repair)
 
       # We will use the mark to mark all objects that we have checked already.
       # Before we start, we need to clear all marks.
       @db.clear_marks
 
-      objects = 0
-      @root_objects.each do |name, id|
-        objects += 1
-        errors += check_object(id, repair)
+      @progressmeter.start("Checking object link structure",
+                           @db.item_counter) do
+        @root_objects.each do |name, id|
+          check_object(id, repair, stats)
+        end
       end
 
       # Delete all broken root objects.
       if repair
         @root_objects.delete_if do |name, id|
-          unless (res = @db.check(id, repair))
+          unless @db.check(id, repair)
             PEROBS.log.error "Discarding broken root object '#{name}' " +
               "with ID #{id}"
-            errors += 1
+            stats[:errors] += 1
           end
-          !res
         end
       end
 
-      if errors > 0
+      if stats[:errors] > 0
         if repair
-          PEROBS.log.error "#{errors} errors found in #{objects} objects"
+          PEROBS.log.error "#{stats[:errors]} errors found in " +
+            "#{stats[:object_cnt]} objects"
         else
-          PEROBS.log.fatal "#{errors} errors found in #{objects} objects"
+          PEROBS.log.fatal "#{stats[:errors]} errors found in " +
+            "#{stats[:object_cnt]} objects"
         end
       else
         PEROBS.log.debug "No errors found"
@@ -410,7 +445,7 @@ module PEROBS
       # Ensure that any fixes are written into the DB.
       sync if repair
 
-      errors
+      stats[:errors]
     end
 
     # This method will execute the provided block as an atomic transaction
@@ -420,35 +455,40 @@ module PEROBS
     # beginning of the transaction. The exception is passed on to the
     # enclosing scope, so you probably want to handle it accordingly.
     def transaction
-      @cache.begin_transaction
+      @lock.synchronize { @cache.begin_transaction }
       begin
         yield if block_given?
       rescue => e
-        @cache.abort_transaction
+        @lock.synchronize { @cache.abort_transaction }
         raise e
       end
-      @cache.end_transaction
+      @lock.synchronize { @cache.end_transaction }
     end
 
     # Calls the given block once for each object, passing that object as a
     # parameter.
     def each
-      @db.clear_marks
-      # Start with the object 0 and the indexes of the root objects. Push them
-      # onto the work stack.
-      stack = [ 0 ] + @root_objects.values
-      while !stack.empty?
-        # Get an object index from the stack.
-        unless (obj = object_by_id(id = stack.pop))
-          PEROBS.log.fatal "Database is corrupted. Object with ID #{id} " +
-            "not found."
-        end
-        # Mark the object so it will never be pushed to the stack again.
-        @db.mark(id)
-        yield(obj.myself) if block_given?
-        # Push the IDs of all unmarked referenced objects onto the stack
-        obj._referenced_object_ids.each do |r_id|
-          stack << r_id unless @db.is_marked?(r_id)
+      @lock.synchronize do
+        @db.clear_marks
+        # Start with the object 0 and the indexes of the root objects. Push them
+        # onto the work stack.
+        stack = [ 0 ] + @root_objects.values
+        while !stack.empty?
+          # Get an object index from the stack.
+          id = stack.pop
+          next if @db.is_marked?(id)
+
+          unless (obj = object_by_id_internal(id))
+            PEROBS.log.fatal "Database is corrupted. Object with ID #{id} " +
+              "not found."
+          end
+          # Mark the object so it will never be pushed to the stack again.
+          @db.mark(id)
+          yield(obj.myself) if block_given?
+          # Push the IDs of all unmarked referenced objects onto the stack
+          obj._referenced_object_ids.each do |r_id|
+            stack << r_id unless @db.is_marked?(r_id)
+          end
         end
       end
     end
@@ -456,7 +496,7 @@ module PEROBS
     # Rename classes of objects stored in the data base.
     # @param rename_map [Hash] Hash that maps the old name to the new name
     def rename_classes(rename_map)
-      @class_map.rename(rename_map)
+      @lock.synchronize { @class_map.rename(rename_map) }
     end
 
     # Internal method. Don't use this outside of this library!
@@ -464,14 +504,16 @@ module PEROBS
     # random numbers between 0 and 2**64 - 1.
     # @return [Integer]
     def _new_id
-      begin
-        # Generate a random number. It's recommended to not store more than
-        # 2**62 objects in the same store.
-        id = rand(2**64)
-        # Ensure that we don't have already another object with this ID.
-      end while @in_memory_objects.include?(id) || @db.include?(id)
+      @lock.synchronize do
+        begin
+          # Generate a random number. It's recommended to not store more than
+          # 2**62 objects in the same store.
+          id = rand(2**64)
+          # Ensure that we don't have already another object with this ID.
+        end while @in_memory_objects.include?(id) || @db.include?(id)
 
-      id
+        id
+      end
     end
 
     # Internal method. Don't use this outside of this library!
@@ -482,7 +524,18 @@ module PEROBS
     # @param obj [BasicObject] Object to register
     # @param id [Integer] object ID
     def _register_in_memory(obj, id)
-      @in_memory_objects[id] = obj.object_id
+      @lock.synchronize do
+        unless obj.is_a?(ObjectBase)
+          PEROBS.log.fatal "You can only register ObjectBase objects"
+        end
+        if @in_memory_objects.include?(id)
+          PEROBS.log.fatal "The Store::_in_memory_objects list already " +
+            "contains an object for ID #{id}"
+        end
+
+        @in_memory_objects[id] = obj.object_id
+        @stats[:created_objects] += 1
+      end
     end
 
     # Remove the object from the in-memory list. This is an internal method
@@ -490,40 +543,101 @@ module PEROBS
     # finalizer, so many restrictions apply!
     # @param id [Integer] Object ID of object to remove from the list
     def _collect(id, ruby_object_id)
+      # This method should only be called from the Ruby garbage collector.
+      # Therefor no locking is needed or even possible. The GC can kick in at
+      # any time and we could be anywhere in the code. So there is a small
+      # risk for a race here, but it should not have any serious consequences.
       if @in_memory_objects[id] == ruby_object_id
         @in_memory_objects.delete(id)
+        @stats[:collected_objects] += 1
       end
     end
 
     # This method returns a Hash with some statistics about this store.
     def statistics
-      @stats.in_memory_objects = @in_memory_objects.length
-      @stats.root_objects = @root_objects.length
+      @lock.synchronize do
+        @stats.in_memory_objects = @in_memory_objects.length
+        @stats.root_objects = @root_objects.length
+      end
 
       @stats
     end
 
     private
 
+    def object_by_id_internal(id)
+      if (ruby_object_id = @in_memory_objects[id])
+        # We have the object in memory so we can just return it.
+        begin
+          object = ObjectSpace._id2ref(ruby_object_id)
+          # Let's make sure the object is really the object we are looking
+          # for. The GC might have recycled it already and the Ruby object ID
+          # could now be used for another object.
+          if object.is_a?(ObjectBase) && object._id == id
+            return object
+          end
+        rescue RangeError => e
+          # Due to a race condition the object can still be in the
+          # @in_memory_objects list but has been collected already by the Ruby
+          # GC. The _collect() call has not been completed yet. We now have to
+          # wait until this has been done. I think the GC lock will prevent a
+          # race on @in_memory_objects.
+          GC.start
+          while @in_memory_objects.include?(id)
+            sleep 0.01
+          end
+        end
+      end
+
+      # This is just a safety check. It has never triggered, so we can disable
+      # it for now.
+      #if (obj = @cache.object_by_id(id))
+      #  PEROBS.log.fatal "Object #{id} with Ruby #{obj.object_id} is in " +
+      #    "cache but not in_memory"
+      #end
+
+      # We don't have the object in memory. Let's find it in the storage.
+      if @db.include?(id)
+        # Great, object found. Read it into memory and return it.
+        obj = ObjectBase::read(self, id)
+        # Add the object to the in-memory storage list.
+        @cache.cache_read(obj)
+
+        return obj
+      end
+
+      # The requested object does not exist. Return nil.
+      nil
+    end
+
     # Mark phase of a mark-and-sweep garbage collector. It will mark all
     # objects that are reachable from the root objects.
     def mark
       classes = Set.new
       marked_objects = 0
-      each { |obj| classes.add(obj.class); marked_objects += 1 }
+      @progressmeter.start("Marking linked objects", @db.item_counter) do
+        each do |obj|
+          classes.add(obj.class)
+          @progressmeter.update(marked_objects += 1)
+        end
+      end
       @class_map.keep(classes.map { |c| c.to_s })
 
       # The root_objects object is included in the count, but we only want to
       # count user objects here.
-      PEROBS.log.debug "#{marked_objects - 1} objects marked"
+      PEROBS.log.debug "#{marked_objects - 1} of #{@db.item_counter} " +
+        "objects marked"
       @stats.marked_objects = marked_objects - 1
     end
 
     # Sweep phase of a mark-and-sweep garbage collector. It will remove all
     # unmarked objects from the store.
     def sweep
-      @stats.swept_objects = @db.delete_unmarked_objects.length
-      @cache.reset
+      @stats.swept_objects = @db.delete_unmarked_objects do |id|
+        @cache.evict(id)
+      end
+      @db.clear_marks
+      GC.start
       PEROBS.log.debug "#{@stats.swept_objects} objects collected"
       @stats.swept_objects
     end
@@ -534,8 +648,7 @@ module PEROBS
     #        with
     # @param repair [Boolean] Delete refernces to broken objects if true
     # @return [Integer] The number of references to bad objects.
-    def check_object(start_id, repair)
-      errors = 0
+    def check_object(start_id, repair, stats)
       @db.mark(start_id)
       # The todo list holds a touple for each object that still needs to be
       # checked. The first item is the referring object and the second is the
@@ -546,7 +659,13 @@ module PEROBS
         # Get the next PEROBS object to check
         ref_obj, id = todo_list.pop
 
-        if (obj = object_by_id(id))
+        begin
+          obj = object_by_id(id)
+        rescue PEROBS::FatalError
+          obj = nil
+        end
+
+        if obj
           # The object exists and is OK. Mark is as checked.
           @db.mark(id)
           # Now look at all other objects referenced by this object.
@@ -569,11 +688,11 @@ module PEROBS
                 ref_obj.inspect
             end
           end
-          errors += 1
+          stats[:errors] += 1
         end
-      end
 
-      errors
+        @progressmeter.update(stats[:object_cnt] += 1)
+      end
     end
 
   end

data/lib/perobs/version.rb

@@ -1,4 +1,4 @@
 module PEROBS
   # The version number
-  VERSION = "4.0.0"
+  VERSION = "4.4.0"
 end