perobs 3.0.2 → 4.0.0

data/lib/perobs/PersistentObjectCache.rb

@@ -0,0 +1,153 @@
+# encoding: UTF-8
+#
+# = PersistentObjectCache.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/PersistentObjectCacheLine'
+
+module PEROBS
+
+  class PersistentObjectCache
+
+    FLUSH_WATERMARK = 500
+
+    # This cache class manages the presence of objects that primarily live in
+    # a backing store but temporarily exist in memory as well. To work with
+    # these objects, direct references must be only very short lived. Indirect
+    # references can be done via a unique ID that the object must provide. Due
+    # to the indirect references the Ruby garbage collector can collect these
+    # objects and the cache is notified via a finalizer that the objects must
+    # provide. The finalize must call the _collect() method. To reduce the
+    # read and write latencies of the backing store this class keeps a subset
+    # of the object in memory which prevents them from being collected. All
+    # references to the objects must be resolved via the get() method to
+    # prevent duplicate instances in memory of the same object.
+    # @param size [Integer] Maximum number of objects to be cached at a time
+    # @param klass [Class] The class of the objects to be cached. Objects must
+    #        provide a uid() method that returns a unique ID for every object.
+    # @param collection [] The object collection the objects belong to. It
+    #        must provide a ::load method.
+    def initialize(size, klass, collection)
+      @size = size
+      @klass = klass
+      @collection = collection
+      @flush_counter = FLUSH_WATERMARK
+      clear
+    end
+
+    # Insert an object into the cache.
+    # @param object [Object] Object to cache
+    # @param modified [Boolean] True if the object was modified, false otherwise
+    def insert(object, modified = true)
+      # Store the object via its Ruby object ID instead of a direct reference.
+      # This allows the object to be collected by the garbage collector.
+      @in_memory_objects[object.uid] = object.object_id
+
+      @lines[object.uid % @size].insert(object, modified)
+    end
+
+    # Retrieve a object reference from the cache.
+    # @param uid [Integer] uid of the object to retrieve.
+    def get(uid)
+      if (entry = @lines[uid % @size].get(uid))
+        return entry.obj
+      end
+
+      if (ruby_object_id = @in_memory_objects[uid])
+        # We have the object in memory so we can just return it.
+        begin
+          object = ObjectSpace._id2ref(ruby_object_id)
+          # Let's make sure the object is really the object we are looking
+          # for. The GC might have recycled it already and the Ruby object ID
+          # could now be used for another object.
+          if object.is_a?(@klass) && object.uid == uid
+            # Let's put the object in the cache. We might need it soon again.
+            insert(object, false)
+            return object
+          end
+        rescue RangeError
+          # Due to a race condition the object can still be in the
+          # @in_memory_objects list but has been collected already by the Ruby
+          # GC. In that case we need to load it again. In this case the
+          # _collect() call will happen much later, potentially after we have
+          # registered a new object with the same ID.
+          @in_memory_objects.delete(uid)
+        end
+      end
+
+      @klass::load(@collection, uid)
+    end
+
+    # Remove a object from the cache.
+    # @param uid [Integer] unique ID of object to remove.
+    def delete(uid)
+      # The object is likely still in memory, but we really don't want to
+      # access it anymore.
+      @in_memory_objects.delete(uid)
+
+      @lines[uid % @size].delete(uid)
+    end
+
+    # Remove a object from the in-memory list. This is an internal method
+    # and should never be called from user code. It will be called from a
+    # finalizer, so many restrictions apply!
+    # @param uid [Integer] Object address of the object to remove from
+    #        the list
+    # @param ruby_object_id [Integer] The Ruby object ID of the collected
+    #        object
+    def _collect(address, ruby_object_id)
+      if @in_memory_objects[id] == ruby_object_id
+        @in_memory_objects.delete(address)
+      end
+    end
+
+    # Write all excess modified objects into the backing store. If now is true
+    # all modified objects will be written.
+    # @param now [Boolean]
+    def flush(now = false)
+      if now || (@flush_counter -= 1) <= 0
+        @lines.each { |line| line.flush(now) }
+        @flush_counter = FLUSH_WATERMARK
+      end
+    end
+
+    # Remove all entries from the cache.
+    def clear
+      # A hash that stores all objects by the Ruby object ID that are
+      # currently in memory. Objects are added via insert() and will be
+      # removed via delete() or _collect() called from a Object
+      # finalizer. It only stores the object Ruby object ID hashed by their
+      # address in the file.  This enables them from being collected by the
+      # Ruby garbage collector.
+      @in_memory_objects = {}
+      # This is the actual cache. The Array stores objects as Entry objects to
+      # also store the modified/not-modified state.
+      @lines = ::Array.new(@size) { |i| PersistentObjectCacheLine.new }
+    end
+
+  end
+
+end
+

data/lib/perobs/PersistentObjectCacheLine.rb

@@ -0,0 +1,87 @@
+# encoding: UTF-8
+#
+# = PersistentObjectCacheLine.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module PEROBS
+
+  class PersistentObjectCacheLine
+
+    # Utility class to store persistent objects and their
+    # modified/not-modified state.
+    class Entry < Struct.new(:obj, :modified)
+    end
+
+    WATERMARK = 4
+
+    def initialize
+      @entries = []
+    end
+
+    def insert(object, modified)
+      @entries.each do |e|
+        if e.obj.uid == object.uid
+          e.modified = true if modified
+          return
+        end
+      end
+
+      # Insert the new entry at the beginning of the line.
+      @entries.unshift(Entry.new(object, modified))
+    end
+
+    def get(uid)
+      @entries.each do |e|
+        return e if e.obj.uid == uid
+      end
+
+      nil
+    end
+
+    # Delete the entry that matches the given UID
+    # @param uid [Integer]
+    def delete(uid)
+      @entries.delete_if { |e| e.obj.uid == uid }
+    end
+
+    # Save all modified entries and delete all but the most recently added.
+    def flush(now)
+      if now || @entries.length > WATERMARK
+        @entries.each do |e|
+          if e.modified
+            e.obj.save
+            e.modified = false
+          end
+        end
+
+        # Delete all but the first WATERMARK entry.
+        @entries = @entries[0..WATERMARK - 1] if @entries.length > WATERMARK
+      end
+    end
+
+  end
+
+end
+

data/lib/perobs/SpaceTree.rb

@@ -27,7 +27,7 @@
 
 require 'perobs/Log'
 require 'perobs/EquiBlobsFile'
-require 'perobs/SpaceTreeNodeCache'
+require 'perobs/PersistentObjectCache'
 require 'perobs/SpaceTreeNode'
 require 'perobs/FlatFile'
 
@@ -51,7 +51,9 @@ module PEROBS
       @nodes = EquiBlobsFile.new(@dir, 'database_spaces',
                                  SpaceTreeNode::NODE_BYTES, 1)
 
-      @cache = SpaceTreeNodeCache.new(self, 128)
+      # Benchmark runs showed a cache size of 128 to be a good compromise
+      # between read and write performance trade-offs and memory consumption.
+      @cache = PersistentObjectCache.new(128, SpaceTreeNode, self)
     end
 
     # Open the SpaceTree file.
@@ -66,7 +68,7 @@ module PEROBS
 
     # Close the SpaceTree file.
     def close
-      @cache.flush
+      @cache.flush(true)
       @nodes.close
       @root_address = nil
       @cache.clear

data/lib/perobs/SpaceTreeNode.rb

@@ -76,16 +76,16 @@ module PEROBS
       @larger = larger
 
       ObjectSpace.define_finalizer(
-        self, SpaceTreeNode._finalize(@tree, @node_address))
-      @tree.cache.insert_unmodified(self)
+        self, SpaceTreeNode._finalize(@tree, @node_address, object_id))
+      @tree.cache.insert(self, false)
     end
 
     # This method generates the destructor for the objects of this class. It
     # is done this way to prevent the Proc object hanging on to a reference to
     # self which would prevent the object from being collected. This internal
     # method is not intended for users to call.
-    def SpaceTreeNode._finalize(tree, node_address)
-      proc { tree.cache._collect(node_address) }
+    def SpaceTreeNode._finalize(tree, node_address, ruby_object_id)
+      proc { tree.cache._collect(node_address, ruby_object_id) }
     end
 
     # Create a new SpaceTreeNode. This method should be used for the creation
@@ -135,6 +135,7 @@ module PEROBS
       node
     end
 
+    # Save the node into the blob file.
     def save
       bytes = [ @blob_address, @size,
                 @parent ? @parent.node_address : 0,
@@ -296,7 +297,13 @@ module PEROBS
         PEROBS.log.fatal "Cannot unlink unknown child node with address " +
           "#{child_node.node_address} from #{to_s}"
       end
-      @tree.cache.insert_modified(self)
+      @tree.cache.insert(self)
+    end
+
+    # @return [Integer] The node address since it uniquely identifies the
+    #         Node.
+    def uid
+      @node_address
     end
 
     # Depth-first iterator for all nodes. The iterator yields the given block
@@ -436,7 +443,7 @@ module PEROBS
     def set_size_and_address(size, address)
       @size = size
       @blob_address = address
-      @tree.cache.insert_modified(self)
+      @tree.cache.insert(self)
     end
 
     def set_link(name, node_or_address)
@@ -452,12 +459,12 @@ module PEROBS
         # Clear the node link.
         instance_variable_set(name, nil)
       end
-      @tree.cache.insert_modified(self)
+      @tree.cache.insert(self)
     end
 
     def parent=(p)
       @parent = p ? SpaceTreeNodeLink.new(@tree, p) : nil
-      @tree.cache.insert_modified(self)
+      @tree.cache.insert(self)
     end
     # Compare this node to another node.
     # @return [Boolean] true if node address is identical, false otherwise

data/lib/perobs/Store.rb

@@ -67,12 +67,20 @@ module PEROBS
   #
   # class Person < PEROBS::Object
   #
-  #   po_attr :name, :mother, :father, :kids
+  #   attr_persist :name, :mother, :father, :kids
   #
+  #   # The contructor is only called for the creation of a new object. It is
+  #   # not called when the object is restored from the database. In that case
+  #   # only restore() is called.
   #   def initialize(cf, name)
   #     super(cf)
-  #     attr_init(:name, name)
-  #     attr_init(:kids, @store.new(PEROBS::Array))
+  #     self.name = name
+  #     self.kids = @store.new(PEROBS::Array)
+  #   end
+  #
+  #   def restore
+  #     # In case you need to do any checks or massaging (e. g. for additional
+  #     # attributes) you can provide this method.
   #   end
   #
   #   def to_s
@@ -90,7 +98,7 @@ module PEROBS
   # joe.kids << jim
   # jim.mother = jane
   # jane.kids << jim
-  # store.sync
+  # store.exit
   #
   class Store
 
@@ -126,7 +134,7 @@ module PEROBS
     #                      Unfortunately, it is 10x slower than marshal.
     def initialize(data_base, options = {})
       # Create a backing store handler
-      @db = (options[:engine] || BTreeDB).new(data_base, options)
+      @db = (options[:engine] || FlatFileDB).new(data_base, options)
       @db.open
       # Create a map that can translate classes to numerical IDs and vice
       # versa.
@@ -228,10 +236,12 @@ module PEROBS
     end
 
     # Delete the entire store. The store is no longer usable after this
-    # method was called.
+    # method was called. This is an alternative to exit() that additionaly
+    # deletes the entire database.
     def delete_store
       @db.delete_database
-      @db = @class_map = @cache = @root_objects = nil
+      @db = @class_map = @in_memory_objects = @stats = @cache = @root_objects =
+        nil
     end
 
     # Store the provided object under the given name. Use this to make the
@@ -310,18 +320,30 @@ module PEROBS
     # public API and should never be called by outside users. It's purely
     # intended for internal use.
     def object_by_id(id)
-      if (obj = @in_memory_objects[id])
+      if (ruby_object_id = @in_memory_objects[id])
         # We have the object in memory so we can just return it.
         begin
-          return ObjectSpace._id2ref(obj)
-        rescue RangeError
+          object = ObjectSpace._id2ref(ruby_object_id)
+          # Let's make sure the object is really the object we are looking
+          # for. The GC might have recycled it already and the Ruby object ID
+          # could now be used for another object.
+          if object.is_a?(ObjectBase) && object._id == id
+            return object
+          end
+        rescue RangeError => e
           # Due to a race condition the object can still be in the
           # @in_memory_objects list but has been collected already by the Ruby
-          # GC. In that case we need to load it again.
+          # GC. In that case we need to load it again. In this case the
+          # _collect() call will happen much later, potentially after we have
+          # registered a new object with the same ID.
           @in_memory_objects.delete(id)
         end
       end
 
+      if (obj = @cache.object_by_id(id))
+        PEROBS.log.fatal "Object #{id} with Ruby #{obj.object_id} is in cache but not in_memory"
+      end
+
       # We don't have the object in memory. Let's find it in the storage.
       if @db.include?(id)
         # Great, object found. Read it into memory and return it.
@@ -332,6 +354,10 @@ module PEROBS
         return obj
       end
 
+      #if (obj = @db.search_object(id))
+      #  PEROBS.log.fatal "Object was not in index but in DB"
+      #end
+
       # The requested object does not exist. Return nil.
       nil
     end
@@ -463,8 +489,10 @@ module PEROBS
     # and should never be called from user code. It will be called from a
     # finalizer, so many restrictions apply!
     # @param id [Integer] Object ID of object to remove from the list
-    def _collect(id, ignore_errors = false)
-      @in_memory_objects.delete(id)
+    def _collect(id, ruby_object_id)
+      if @in_memory_objects[id] == ruby_object_id
+        @in_memory_objects.delete(id)
+      end
     end
 
     # This method returns a Hash with some statistics about this store.