perobs 4.3.0 → 4.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a61fc945e0ef9f5ed6558080931d2acae42cc0401f375275684e4ee32fefe4f7
-  data.tar.gz: 4d864fdc0791aa78d8c180b4686ee825cd25e209284fca1966f144813c063280
+  metadata.gz: c2c526c97aab15c09f8e8acbcb432203e7d60fef8c504df6ddb0b8f35459cca0
+  data.tar.gz: ad077bb879c041289c7a474e87645064ff84203c674640aff59b5f23923ac1b7
 SHA512:
-  metadata.gz: f3834a9caae693d82837fb9f75141cb35e85f1a2c1439d1bb898f8578d9ae082f46deb0233d2b8a02d6ae6b0bf66862098b47ff571ff3d9a6b874fadaef6d23a
-  data.tar.gz: 883f1b5e553fae2be0039aa090d89bf6eb44ec1d0dc31488aeaa727ec8bc2844c9b722568cd7dde0b33c507121afd11e720154d4ff27e66aa3ac3812d5603954
+  metadata.gz: b86fc662ef8bbf623ee7da777f023a74e3db9502df18c45dcda9ce244982225aaec7e74ce25378147c807775765c57d801810ac16be01d2145227bb0642caa3b
+  data.tar.gz: d0c5a79eb60a1221bc385a1fafbc77a1aaa1e47782b3ea27ca92f5f7c539804bc00a388fdddc1ab956d7805dbdd52d055f5cdd76a8bb155ec5ada4335b7b9175

data/README.md

@@ -6,11 +6,12 @@ them from PEROBS::Object. They will be in memory when needed and
 transparently stored into a persistent storage.
 
 This library is ideal for Ruby applications that work on huge, mostly
-constant data sets and usually handle a small subset of the data at a
+static data sets and usually process a small subset of the data at a
 time. To ensure data consistency of a larger data set, you can use
 transactions to make modifications of multiple objects atomicaly.
 Transactions can be nested and are aborted when an exception is
-raised.
+raised. PEROBS is thread-safe, so you can use it in a multi-threaded
+application.
 
 ## Usage
 
@@ -120,15 +121,18 @@ class Person < PEROBS::Object
 
 end
 
-store = PEROBS::Store.new('family')
-store['grandpa'] = joe = store.new(Person, 'Joe')
-store['grandma'] = jane = store.new(Person, 'Jane')
-jim = store.new(Person, 'Jim')
-jim.father = joe
-joe.kids << jim
-jim.mother = jane
-jane.kids << jim
-store.exit
+begin
+  store = PEROBS::Store.new('family')
+  store['grandpa'] = joe = store.new(Person, 'Joe')
+  store['grandma'] = jane = store.new(Person, 'Jane')
+  jim = store.new(Person, 'Jim')
+  jim.father = joe
+  joe.kids << jim
+  jim.mother = jane
+  jane.kids << jim
+ensure
+  store.exit
+end
 ```
 
 When you run this script, a folder named 'family' will be created. It
@@ -166,9 +170,15 @@ object to another object.
 
 ### Caveats and known issues
 
-PEROBS is currently not thread-safe. You cannot simultaneously access
-the database from multiple application. The library uses locks to
-ensure that only one Store object is accessing the database at a time.
+You cannot simultaneously access the database from multiple
+applications concurrently. The library uses locks to ensure that only
+one Store object is accessing the database at a time.
+
+In case the application terminates without calling Store::exit(), the
+database or the database index could get corrupted. To check the
+consistency of your database you can use Store::check(). To check and
+repair the database you can call Store::repair(). Depending on the
+size of your database, these operations can last minutes to hours.
 
 ## Installation
 
@@ -188,7 +198,8 @@ Or install it yourself as:
 
 ## Copyright and License
 
-Copyright (c) 2015, 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+Copyright (c) 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 by
+Chris Schlaeger <chris@taskjuggler.org>
 
 PEROBS and all accompanying files are licensed under this MIT License
 

data/lib/perobs/BTreeNode.rb

@@ -78,7 +78,7 @@ module PEROBS
       end
     end
 
-    # Create a new SpaceTreeNode. This method should be used for the creation
+    # Create a new BTreeNode. This method should be used for the creation
     # of new nodes instead of calling the constructor directly.
     # @param tree [BTree] The tree the new node should belong to
     # @param parent [BTreeNode] The parent node
@@ -130,7 +130,8 @@ module PEROBS
       ary = bytes.unpack(BTreeNode::node_bytes_format(tree))
       # Read is_leaf
       if ary[0] != 0 && ary[0] != 1
-        PEROBS.log.fatal "First byte of a BTreeNode entry must be 0 or 1"
+        PEROBS.log.fatal "First byte of a BTreeNode entry at address " +
+          "#{address} must be 0 or 1 but is #{ary[0]}"
       end
       is_leaf = ary[0] == 0 ? false : true
       # This is the number of keys this node has.
@@ -350,6 +351,7 @@ module PEROBS
       unless @parent
         # The node is the root node. We need to create a parent node first.
         self.parent = link(BTreeNode::create(@tree, nil, false))
+        @tree.node_cache.insert(self)
         @parent.set_child(0, self)
         @tree.set_root(@parent)
       end
@@ -382,6 +384,7 @@ module PEROBS
       end
 
       i = search_key_index(key)
+      @tree.node_cache.insert(self)
       if @keys[i] == key
         # Overwrite existing entries
         @keys[i] = key
@@ -390,7 +393,6 @@ module PEROBS
         else
           @children[i + 1] = link(value_or_child)
         end
-        @tree.node_cache.insert(self)
 
         return false
       else
@@ -401,7 +403,6 @@ module PEROBS
         else
           @children.insert(i + 1, link(value_or_child))
         end
-        @tree.node_cache.insert(self)
 
         return true
       end
@@ -417,8 +418,8 @@ module PEROBS
       update_branch_key(key) if index == 0
 
       # Delete the corresponding value.
-      removed_value = @values.delete_at(index)
       @tree.node_cache.insert(self)
+      removed_value = @values.delete_at(index)
 
       if @keys.length < min_keys
         if @prev_sibling && @prev_sibling.parent == @parent
@@ -481,9 +482,9 @@ module PEROBS
         PEROBS.log.fatal "Leaf nodes are too big to merge"
       end
 
+      @tree.node_cache.insert(self)
       @keys += node.keys
       @values += node.values
-      @tree.node_cache.insert(self)
 
       node.parent.remove_child(node)
     end
@@ -494,11 +495,11 @@ module PEROBS
       end
 
       index = @parent.search_node_index(node) - 1
+      @tree.node_cache.insert(self)
       @keys << @parent.keys[index]
       @keys += node.keys
       node.children.each { |c| c.parent = link(self) }
       @children += node.children
-      @tree.node_cache.insert(self)
 
       node.parent.remove_child(node)
     end
@@ -525,6 +526,7 @@ module PEROBS
           "#{dest_node.is_leaf} node must be of same kind"
       end
 
+      @tree.node_cache.insert(dest_node)
       dest_node.keys[dst_idx, count] = @keys[src_idx, count]
       if @is_leaf
         # For leaves we copy the keys and corresponding values.
@@ -537,17 +539,17 @@ module PEROBS
           dest_node.set_child(dst_idx + i, @children[src_idx + i])
         end
       end
-      @tree.node_cache.insert(dest_node)
     end
 
     def parent=(p)
-      @parent = p
       @tree.node_cache.insert(self)
+      @parent = p
 
       p
     end
 
     def prev_sibling=(node)
+      @tree.node_cache.insert(self)
       @prev_sibling = node
       if node.nil? && @is_leaf
         # If this node is a leaf node without a previous sibling we need to
@@ -555,14 +557,12 @@ module PEROBS
         @tree.set_first_leaf(BTreeNodeLink.new(@tree, self))
       end
 
-      @tree.node_cache.insert(self)
-
       node
     end
 
     def next_sibling=(node)
-      @next_sibling = node
       @tree.node_cache.insert(self)
+      @next_sibling = node
       if node.nil? && @is_leaf
         # If this node is a leaf node without a next sibling we need to
         # register it as the last leaf node.
@@ -573,25 +573,25 @@ module PEROBS
     end
 
     def set_child(index, child)
+      @tree.node_cache.insert(self)
       if child
         @children[index] = link(child)
         @children[index].parent = link(self)
       else
         @children[index] = nil
       end
-      @tree.node_cache.insert(self)
 
       child
     end
 
     def trim(idx)
+      @tree.node_cache.insert(self)
       @keys.slice!(idx, @keys.length - idx)
       if @is_leaf
         @values.slice!(idx, @values.length - idx)
       else
         @children.slice!(idx + 1, @children.length - idx - 1)
       end
-      @tree.node_cache.insert(self)
     end
 
     # Search the keys of the node that fits the given key. The result is

data/lib/perobs/BigArrayNode.rb

@@ -610,7 +610,7 @@ module PEROBS
       end
     end
 
-    # @param index [offset] offset to search the child index for
+    # @param offset [Integer] offset to search the child index for
     # @return [Integer] Index of the matching offset or the insert position.
     def search_child_index(offset)
       # Handle special case for empty offsets list.

data/lib/perobs/Cache.rb

@@ -122,12 +122,25 @@ module PEROBS
     # @param id [Integer] ID of the cached PEROBS::ObjectBase
     def object_by_id(id)
       idx = id & @mask
-      # The index is just a hash. We still need to check if the object IDs are
-      # actually the same before we can return the object.
-      if (obj = @writes[idx]) && obj._id == id
-        # The object was in the write cache.
-        return obj
-      elsif (obj = @reads[idx]) && obj._id == id
+
+      if @transaction_stack.empty?
+        # The index is just a hash. We still need to check if the object IDs are
+        # actually the same before we can return the object.
+        if (obj = @writes[idx]) && obj._id == id
+          # The object was in the write cache.
+          return obj
+        end
+      else
+        # During transactions, the read cache is used to provide fast access
+        # to modified objects. But it does not store all modified objects
+        # since there can be hash collisions. So we also have to check all
+        # transaction objects first.
+        if (obj = @transaction_objects[id])
+          return obj
+        end
+      end
+
+      if (obj = @reads[idx]) && obj._id == id
         # The object was in the read cache.
         return obj
       end

data/lib/perobs/EquiBlobsFile.rb

@@ -425,6 +425,8 @@ module PEROBS
                    pack("Q#{@custom_data_values.length}"))
         end
         @f.flush
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot write EquiBlobsFile header: " + e.message
       end
     end
 

data/lib/perobs/FlatFile.rb

@@ -221,6 +221,7 @@ module PEROBS
         flags |= (1 << FlatFileBlobHeader::COMPRESSED_FLAG_BIT) if compressed
         FlatFileBlobHeader.new(@f, addr, flags, raw_obj_bytesize, id, crc).write
         @f.write(raw_obj)
+        @f.flush
         if length != -1 && raw_obj_bytesize < length
           # The new object was not appended and it did not completely fill the
           # free space. So we have to write a new header to mark the remaining
@@ -247,12 +248,11 @@ module PEROBS
           # If we had an existing object stored for the ID we have to mark
           # this entry as deleted now.
           old_header.clear_flags
+          @f.flush
           # And register the newly freed space with the space list.
           if @space_list.is_open?
             @space_list.add_space(old_addr, old_header.length)
           end
-        else
-          @f.flush
         end
       rescue IOError => e
         PEROBS.log.fatal "Cannot write blob for ID #{id} to FlatFileDB: " +
@@ -355,7 +355,7 @@ module PEROBS
       valid_blobs = 0
 
       # Iterate over all entries.
-      @progressmeter.start('Defragmentizing blobs file', @f.size) do |pm|
+      @progressmeter.start('Defragmenting blobs file', @f.size) do |pm|
         each_blob_header do |header|
           # If we have stumbled over a corrupted blob we treat it similar to a
           # deleted blob and reuse the space.
@@ -578,7 +578,6 @@ module PEROBS
     # Repair the FlatFile. In contrast to the repair functionality in the
     # check() method this method is much faster. It simply re-creates the
     # index and space list from the blob file.
-    # @param repair [Boolean] True if errors should be fixed.
     # @return [Integer] Number of errors found
     def repair
       errors = 0

data/lib/perobs/IDListPageRecord.rb

@@ -65,7 +65,7 @@ module PEROBS
     end
 
     # Insert an ID into the page.
-    # @param ID [Integer] The ID to store
+    # @param id [Integer] The ID to store
     def insert(id)
       unless @min_id <= id && id <= @max_id
         raise ArgumentError, "IDs for this page must be between #{@min_id} " +

data/lib/perobs/Store.rb

@@ -2,7 +2,7 @@
 #
 # = Store.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016, 2017, 2018, 2019
+# Copyright (c) 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022
 # by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
@@ -27,6 +27,7 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 require 'set'
+require 'monitor'
 
 require 'perobs/Log'
 require 'perobs/Handle'
@@ -46,7 +47,7 @@ require 'perobs/ConsoleProgressMeter'
 # PErsistent Ruby OBject Store
 module PEROBS
 
-  Statistics = Struct.new(:in_memory_objects, :root_objects, :zombie_objects,
+  Statistics = Struct.new(:in_memory_objects, :root_objects,
                           :marked_objects, :swept_objects,
                           :created_objects, :collected_objects)
 
@@ -160,9 +161,6 @@ module PEROBS
       # List of PEROBS objects that are currently available as Ruby objects
       # hashed by their ID.
       @in_memory_objects = {}
-      # List of objects that were destroyed already but were still found in
-      # the in_memory_objects list. _collect has not yet been called for them.
-      @zombie_objects = {}
 
       # This objects keeps some counters of interest.
       @stats = Statistics.new
@@ -173,6 +171,9 @@ module PEROBS
       # objects in memory.
       @cache = Cache.new(options[:cache_bits] || 16)
 
+      # Lock to serialize access to the Store and all stored data.
+      @lock = Monitor.new
+
       # The named (global) objects IDs hashed by their name
       unless options[:no_root_objects]
         unless (@root_objects = object_by_id(0))
@@ -243,8 +244,8 @@ module PEROBS
         end
       end
 
-      @db = @class_map = @in_memory_objects = @zombie_objects =
-        @stats = @cache = @root_objects = nil
+      @db = @class_map = @in_memory_objects = @stats = @cache =
+        @root_objects = nil
     end
 
     # You need to call this method to create new PEROBS objects that belong to
@@ -259,11 +260,13 @@ module PEROBS
         PEROBS.log.fatal "#{klass} is not a BasicObject derivative"
       end
 
-      obj = _construct_po(klass, _new_id, *args)
-      # Mark the new object as modified so it gets pushed into the database.
-      @cache.cache_write(obj)
-      # Return a POXReference proxy for the newly created object.
-      obj.myself
+      @lock.synchronize do
+        obj = _construct_po(klass, _new_id, *args)
+        # Mark the new object as modified so it gets pushed into the database.
+        @cache.cache_write(obj)
+        # Return a POXReference proxy for the newly created object.
+        obj.myself
+      end
     end
 
     # For library internal use only!
@@ -280,9 +283,11 @@ module PEROBS
     # method was called. This is an alternative to exit() that additionaly
     # deletes the entire database.
     def delete_store
-      @db.delete_database
-      @db = @class_map = @in_memory_objects = @zombie_objects =
-        @stats = @cache = @root_objects = nil
+      @lock.synchronize do
+        @db.delete_database
+        @db = @class_map = @in_memory_objects = @stats = @cache =
+          @root_objects = nil
+      end
     end
 
     # Store the provided object under the given name. Use this to make the
@@ -294,25 +299,27 @@ module PEROBS
     # @param obj [PEROBS::Object] The object to store
     # @return [PEROBS::Object] The stored object.
     def []=(name, obj)
-      # If the passed object is nil, we delete the entry if it exists.
-      if obj.nil?
-        @root_objects.delete(name)
-        return nil
-      end
+      @lock.synchronize do
+        # If the passed object is nil, we delete the entry if it exists.
+        if obj.nil?
+          @root_objects.delete(name)
+          return nil
+        end
 
-      # We only allow derivatives of PEROBS::Object to be stored in the
-      # store.
-      unless obj.is_a?(ObjectBase)
-        PEROBS.log.fatal 'Object must be of class PEROBS::Object but ' +
-          "is of class #{obj.class}"
-      end
+        # We only allow derivatives of PEROBS::Object to be stored in the
+        # store.
+        unless obj.is_a?(ObjectBase)
+          PEROBS.log.fatal 'Object must be of class PEROBS::Object but ' +
+            "is of class #{obj.class}"
+        end
 
-      unless obj.store == self
-        PEROBS.log.fatal 'The object does not belong to this store.'
-      end
+        unless obj.store == self
+          PEROBS.log.fatal 'The object does not belong to this store.'
+        end
 
-      # Store the name and mark the name list as modified.
-      @root_objects[name] = obj._id
+        # Store the name and mark the name list as modified.
+        @root_objects[name] = obj._id
+      end
 
       obj
     end
@@ -322,28 +329,34 @@ module PEROBS
     #        returned.
     # @return The requested object or nil if it doesn't exist.
     def [](name)
-      # Return nil if there is no object with that name.
-      return nil unless (id = @root_objects[name])
+      @lock.synchronize do
+        # Return nil if there is no object with that name.
+        return nil unless (id = @root_objects[name])
 
-      POXReference.new(self, id)
+        POXReference.new(self, id)
+      end
     end
 
     # Return a list with all the names of the root objects.
     # @return [Array of Symbols]
     def names
-      @root_objects.keys
+      @lock.synchronize do
+        @root_objects.keys
+      end
     end
 
     # Flush out all modified objects to disk and shrink the in-memory list if
     # needed.
     def sync
-      if @cache.in_transaction?
-        @cache.abort_transaction
+      @lock.synchronize do
+        if @cache.in_transaction?
+          @cache.abort_transaction
+          @cache.flush
+          PEROBS.log.fatal "You cannot call sync() during a transaction: \n" +
+            Kernel.caller.join("\n")
+        end
         @cache.flush
-        PEROBS.log.fatal "You cannot call sync() during a transaction: \n" +
-          Kernel.caller.join("\n")
       end
-      @cache.flush
     end
 
     # Return the number of object stored in the store. CAVEAT: This method
@@ -353,7 +366,9 @@ module PEROBS
     def size
       # We don't include the Hash that stores the root objects into the object
       # count.
-      @db.item_counter - 1
+      @lock.synchronize do
+        @db.item_counter - 1
+      end
     end
 
     # Discard all objects that are not somehow connected to the root objects
@@ -362,51 +377,20 @@ module PEROBS
     # method periodically.
     # @return [Integer] The number of collected objects
     def gc
-      sync
-      mark
-      sweep
+      @lock.synchronize do
+        sync
+        mark
+        sweep
+      end
     end
 
     # Return the object with the provided ID. This method is not part of the
     # public API and should never be called by outside users. It's purely
     # intended for internal use.
     def object_by_id(id)
-      if (ruby_object_id = @in_memory_objects[id])
-        # We have the object in memory so we can just return it.
-        begin
-          object = ObjectSpace._id2ref(ruby_object_id)
-          # Let's make sure the object is really the object we are looking
-          # for. The GC might have recycled it already and the Ruby object ID
-          # could now be used for another object.
-          if object.is_a?(ObjectBase) && object._id == id
-            return object
-          end
-        rescue RangeError => e
-          # Due to a race condition the object can still be in the
-          # @in_memory_objects list but has been collected already by the Ruby
-          # GC. In that case we need to load it again. The _collect() call
-          # will happen much later, potentially after we have registered a new
-          # object with the same ID.
-          @zombie_objects[id] = @in_memory_objects.delete(id)
-        end
-      end
-
-      if (obj = @cache.object_by_id(id))
-        PEROBS.log.fatal "Object #{id} with Ruby #{obj.object_id} is in cache but not in_memory"
+      @lock.synchronize do
+        object_by_id_internal(id)
       end
-
-      # We don't have the object in memory. Let's find it in the storage.
-      if @db.include?(id)
-        # Great, object found. Read it into memory and return it.
-        obj = ObjectBase::read(self, id)
-        # Add the object to the in-memory storage list.
-        @cache.cache_read(obj)
-
-        return obj
-      end
-
-      # The requested object does not exist. Return nil.
-      nil
     end
 
     # This method can be used to check the database and optionally repair it.
@@ -471,38 +455,40 @@ module PEROBS
     # beginning of the transaction. The exception is passed on to the
     # enclosing scope, so you probably want to handle it accordingly.
     def transaction
-      @cache.begin_transaction
+      @lock.synchronize { @cache.begin_transaction }
       begin
         yield if block_given?
       rescue => e
-        @cache.abort_transaction
+        @lock.synchronize { @cache.abort_transaction }
         raise e
       end
-      @cache.end_transaction
+      @lock.synchronize { @cache.end_transaction }
     end
 
     # Calls the given block once for each object, passing that object as a
     # parameter.
     def each
-      @db.clear_marks
-      # Start with the object 0 and the indexes of the root objects. Push them
-      # onto the work stack.
-      stack = [ 0 ] + @root_objects.values
-      while !stack.empty?
-        # Get an object index from the stack.
-        id = stack.pop
-        next if @db.is_marked?(id)
-
-        unless (obj = object_by_id(id))
-          PEROBS.log.fatal "Database is corrupted. Object with ID #{id} " +
-            "not found."
-        end
-        # Mark the object so it will never be pushed to the stack again.
-        @db.mark(id)
-        yield(obj.myself) if block_given?
-        # Push the IDs of all unmarked referenced objects onto the stack
-        obj._referenced_object_ids.each do |r_id|
-          stack << r_id unless @db.is_marked?(r_id)
+      @lock.synchronize do
+        @db.clear_marks
+        # Start with the object 0 and the indexes of the root objects. Push them
+        # onto the work stack.
+        stack = [ 0 ] + @root_objects.values
+        while !stack.empty?
+          # Get an object index from the stack.
+          id = stack.pop
+          next if @db.is_marked?(id)
+
+          unless (obj = object_by_id_internal(id))
+            PEROBS.log.fatal "Database is corrupted. Object with ID #{id} " +
+              "not found."
+          end
+          # Mark the object so it will never be pushed to the stack again.
+          @db.mark(id)
+          yield(obj.myself) if block_given?
+          # Push the IDs of all unmarked referenced objects onto the stack
+          obj._referenced_object_ids.each do |r_id|
+            stack << r_id unless @db.is_marked?(r_id)
+          end
         end
       end
     end
@@ -510,7 +496,7 @@ module PEROBS
     # Rename classes of objects stored in the data base.
     # @param rename_map [Hash] Hash that maps the old name to the new name
     def rename_classes(rename_map)
-      @class_map.rename(rename_map)
+      @lock.synchronize { @class_map.rename(rename_map) }
     end
 
     # Internal method. Don't use this outside of this library!
@@ -518,14 +504,16 @@ module PEROBS
     # random numbers between 0 and 2**64 - 1.
     # @return [Integer]
     def _new_id
-      begin
-        # Generate a random number. It's recommended to not store more than
-        # 2**62 objects in the same store.
-        id = rand(2**64)
-        # Ensure that we don't have already another object with this ID.
-      end while @in_memory_objects.include?(id) || @db.include?(id)
+      @lock.synchronize do
+        begin
+          # Generate a random number. It's recommended to not store more than
+          # 2**62 objects in the same store.
+          id = rand(2**64)
+          # Ensure that we don't have already another object with this ID.
+        end while @in_memory_objects.include?(id) || @db.include?(id)
 
-      id
+        id
+      end
     end
 
     # Internal method. Don't use this outside of this library!
@@ -536,16 +524,18 @@ module PEROBS
     # @param obj [BasicObject] Object to register
     # @param id [Integer] object ID
     def _register_in_memory(obj, id)
-      unless obj.is_a?(ObjectBase)
-        PEROBS.log.fatal "You can only register ObjectBase objects"
-      end
-      if @in_memory_objects.include?(id)
-        PEROBS.log.fatal "The Store::_in_memory_objects list already " +
-          "contains an object for ID #{id}"
-      end
+      @lock.synchronize do
+        unless obj.is_a?(ObjectBase)
+          PEROBS.log.fatal "You can only register ObjectBase objects"
+        end
+        if @in_memory_objects.include?(id)
+          PEROBS.log.fatal "The Store::_in_memory_objects list already " +
+            "contains an object for ID #{id}"
+        end
 
-      @in_memory_objects[id] = obj.object_id
-      @stats[:created_objects] += 1
+        @in_memory_objects[id] = obj.object_id
+        @stats[:created_objects] += 1
+      end
     end
 
     # Remove the object from the in-memory list. This is an internal method
@@ -553,26 +543,73 @@ module PEROBS
     # finalizer, so many restrictions apply!
     # @param id [Integer] Object ID of object to remove from the list
     def _collect(id, ruby_object_id)
+      # This method should only be called from the Ruby garbage collector.
+      # Therefor no locking is needed or even possible. The GC can kick in at
+      # any time and we could be anywhere in the code. So there is a small
+      # risk for a race here, but it should not have any serious consequences.
       if @in_memory_objects[id] == ruby_object_id
         @in_memory_objects.delete(id)
         @stats[:collected_objects] += 1
-      elsif @zombie_objects[id] == ruby_object_id
-        @zombie_objects.delete(id)
-        @stats[:collected_objects] += 1
       end
     end
 
     # This method returns a Hash with some statistics about this store.
     def statistics
-      @stats.in_memory_objects = @in_memory_objects.length
-      @stats.root_objects = @root_objects.length
-      @stats.zombie_objects = @zombie_objects.length
+      @lock.synchronize do
+        @stats.in_memory_objects = @in_memory_objects.length
+        @stats.root_objects = @root_objects.length
+      end
 
       @stats
     end
 
     private
 
+    def object_by_id_internal(id)
+      if (ruby_object_id = @in_memory_objects[id])
+        # We have the object in memory so we can just return it.
+        begin
+          object = ObjectSpace._id2ref(ruby_object_id)
+          # Let's make sure the object is really the object we are looking
+          # for. The GC might have recycled it already and the Ruby object ID
+          # could now be used for another object.
+          if object.is_a?(ObjectBase) && object._id == id
+            return object
+          end
+        rescue RangeError => e
+          # Due to a race condition the object can still be in the
+          # @in_memory_objects list but has been collected already by the Ruby
+          # GC. The _collect() call has not been completed yet. We now have to
+          # wait until this has been done. I think the GC lock will prevent a
+          # race on @in_memory_objects.
+          GC.start
+          while @in_memory_objects.include?(id)
+            sleep 0.01
+          end
+        end
+      end
+
+      # This is just a safety check. It has never triggered, so we can disable
+      # it for now.
+      #if (obj = @cache.object_by_id(id))
+      #  PEROBS.log.fatal "Object #{id} with Ruby #{obj.object_id} is in " +
+      #    "cache but not in_memory"
+      #end
+
+      # We don't have the object in memory. Let's find it in the storage.
+      if @db.include?(id)
+        # Great, object found. Read it into memory and return it.
+        obj = ObjectBase::read(self, id)
+        # Add the object to the in-memory storage list.
+        @cache.cache_read(obj)
+
+        return obj
+      end
+
+      # The requested object does not exist. Return nil.
+      nil
+    end
+
     # Mark phase of a mark-and-sweep garbage collector. It will mark all
     # objects that are reachable from the root objects.
     def mark

data/lib/perobs/version.rb

@@ -1,4 +1,4 @@
 module PEROBS
   # The version number
-  VERSION = "4.3.0"
+  VERSION = "4.4.0"
 end

data/perobs.gemspec

@@ -20,5 +20,5 @@ GEM_SPEC = Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 2.3'
   spec.add_development_dependency 'yard', '~>0.9.12'
-  spec.add_development_dependency 'rake', '~> 12.3.3'
+  spec.add_development_dependency 'rake', '~> 13.0.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: perobs
 version: !ruby/object:Gem::Version
-  version: 4.3.0
+  version: 4.4.0
 platform: ruby
 authors:
 - Chris Schlaeger
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-14 00:00:00.000000000 Z
+date: 2022-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: 13.0.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: 13.0.3
 description: Library to provide a persistent object store
 email:
 - chris@linux.com
@@ -70,7 +70,6 @@ files:
 - lib/perobs/BTreeBlob.rb
 - lib/perobs/BTreeDB.rb
 - lib/perobs/BTreeNode.rb
-- lib/perobs/BTreeNodeCache.rb
 - lib/perobs/BTreeNodeLink.rb
 - lib/perobs/BigArray.rb
 - lib/perobs/BigArrayNode.rb
@@ -162,7 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.2.32
 signing_key:
 specification_version: 4
 summary: Persistent Ruby Object Store

data/lib/perobs/BTreeNodeCache.rb

@@ -1,109 +0,0 @@
-# encoding: UTF-8
-#
-# = BTree.rb -- Persistent Ruby Object Store
-#
-# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
-#
-# MIT License
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-require 'perobs/BTreeNode'
-
-module PEROBS
-
-  class BTreeNodeCache
-
-    def initialize(tree)
-      @tree = tree
-      clear
-    end
-
-    def get(address)
-      if (node = @modified_nodes[address])
-        return node
-      end
-
-      if (node = @top_nodes[address])
-        return node
-      end
-
-      if (node = @ephemeral_nodes[address])
-        return node
-      end
-
-      BTreeNode::load(@tree, address)
-    end
-
-    def set_root(node)
-      node = node.get_node if node.is_a?(BTreeNodeLink)
-
-      @top_nodes = {}
-      @top_nodes[node.node_address] = node
-    end
-
-    def insert(node, modified = true)
-      unless node
-        PEROBS.log.fatal "nil cannot be cached"
-      end
-      node = node.get_node if node.is_a?(BTreeNodeLink)
-
-      if modified
-        @modified_nodes[node.node_address] = node
-      end
-      @ephemeral_nodes[node.node_address] = node
-
-      if !@top_nodes.include?(node) && node.is_top?
-        @top_nodes[node.node_address] = node
-      end
-    end
-
-    def _collect(address, ruby_object_id)
-      # Just a dummy for now
-    end
-
-    # Remove a node from the cache.
-    # @param address [Integer] address of node to remove.
-    def delete(address)
-      @ephemeral_nodes.delete(address)
-      @top_nodes.delete(address)
-      @modified_nodes.delete(address)
-    end
-
-    # Flush all dirty nodes into the backing store.
-    def flush(now = false)
-      if now || @modified_nodes.size > 1024
-        @modified_nodes.each_value { |node| node.write_node }
-        @modified_nodes = {}
-      end
-      @ephemeral_nodes = {}
-    end
-
-    # Remove all nodes from the cache.
-    def clear
-      @top_nodes = {}
-      @ephemeral_nodes = {}
-      @modified_nodes = {}
-    end
-
-  end
-
-end
-