perobs 4.2.0 → 4.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7790ee42586bb2b8fca115f93ed277c4a8057f7a7027b356baea7b066da953e5
-  data.tar.gz: 110e0710a84ef544a4874cf868ec1662dfc900077d1894b24f53bcdaeaeeed34
+  metadata.gz: 90c77003da7bd628fa753bfe35918d8c702b433ac5113ba9fc206f7afa114f6c
+  data.tar.gz: c38f786cb10a040048b7d08b75c16baf034c116a3f9648ece3af7b50e7898446
 SHA512:
-  metadata.gz: d95d845c7e8bd183f53b60369415bde86cd766c224bcbc2c52c870c60542a786f359a63eecc4e8829055cee6a1674bc952277749da183432b4b3abae7536efbb
-  data.tar.gz: 5fa1712fb01118d955d86396aec87b319c75856f6602ccf1a19f475a3dc64dc65a540e35271e6f2c6c7ee6a10c5cb03da30d43945a0dc264c7b0e48f575269d1
+  metadata.gz: 1fea934c359190c3b171c99e4dbffb3af58ec2f9dd755eedc0acb7db556f0e2471bd3c527a2910e5d8f17cd59b3f1ca4079436bc80bd11ea5d6efa60f55a54e9
+  data.tar.gz: db71320a9672848a3e35088e7a8b3389278c987a1bea905336e3f8bfff6daeb27564724b6f9d681d364684a57ce8593021041626615dace155e1849b0944003b

data/README.md

@@ -6,11 +6,12 @@ them from PEROBS::Object. They will be in memory when needed and
 transparently stored into a persistent storage.
 
 This library is ideal for Ruby applications that work on huge, mostly
-constant data sets and usually handle a small subset of the data at a
+static data sets and usually process a small subset of the data at a
 time. To ensure data consistency of a larger data set, you can use
 transactions to make modifications of multiple objects atomicaly.
 Transactions can be nested and are aborted when an exception is
-raised.
+raised. PEROBS is thread-safe, so you can use it in a multi-threaded
+application.
 
 ## Usage
 
@@ -108,7 +109,7 @@ class Person < PEROBS::Object
     attr_init(:father) do { @store.new(Person, 'Dad') }
   end
 
-  def merry(spouse)
+  def marry(spouse)
     self.spouse = spouse
     self.status = :married
   end
@@ -120,15 +121,18 @@ class Person < PEROBS::Object
 
 end
 
-store = PEROBS::Store.new('family')
-store['grandpa'] = joe = store.new(Person, 'Joe')
-store['grandma'] = jane = store.new(Person, 'Jane')
-jim = store.new(Person, 'Jim')
-jim.father = joe
-joe.kids << jim
-jim.mother = jane
-jane.kids << jim
-store.exit
+begin
+  store = PEROBS::Store.new('family')
+  store['grandpa'] = joe = store.new(Person, 'Joe')
+  store['grandma'] = jane = store.new(Person, 'Jane')
+  jim = store.new(Person, 'Jim')
+  jim.father = joe
+  joe.kids << jim
+  jim.mother = jane
+  jane.kids << jim
+ensure
+  store.exit
+end
 ```
 
 When you run this script, a folder named 'family' will be created. It
@@ -166,9 +170,15 @@ object to another object.
 
 ### Caveats and known issues
 
-PEROBS is currently not thread-safe. You cannot simultaneously access
-the database from multiple application. The library uses locks to
-ensure that only one Store object is accessing the database at a time.
+You cannot simultaneously access the database from multiple
+applications concurrently. The library uses locks to ensure that only
+one Store object is accessing the database at a time.
+
+In case the application terminates without calling Store::exit(), the
+database or the database index could get corrupted. To check the
+consistency of your database you can use Store::check(). To check and
+repair the database you can call Store::repair(). Depending on the
+size of your database, these operations can last minutes to hours.
 
 ## Installation
 
@@ -188,7 +198,8 @@ Or install it yourself as:
 
 ## Copyright and License
 
-Copyright (c) 2015, 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+Copyright (c) 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 by
+Chris Schlaeger <chris@taskjuggler.org>
 
 PEROBS and all accompanying files are licensed under this MIT License
 

data/lib/perobs/BTree.rb

@@ -70,7 +70,7 @@ module PEROBS
       @nodes.register_custom_data('first_leaf')
       @nodes.register_custom_data('last_leaf')
       @nodes.register_custom_data('btree_size')
-      @node_cache = PersistentObjectCache.new(2**16, -1, BTreeNode, self)
+      @node_cache = PersistentObjectCache.new(2**13, 2**13, BTreeNode, self)
       @root = @first_leaf = @last_leaf = nil
       @size = 0
 
@@ -190,7 +190,7 @@ module PEROBS
         "Number of leave nodes: #{stats.leave_nodes}; " +
         "Number of leaves: #{stats.leaves}"
 
-      !stats.nil?
+      true
     end
 
     # Register a new node as root node of the tree.

data/lib/perobs/BTreeNode.rb

@@ -59,7 +59,7 @@ module PEROBS
     #        if not
     def initialize(tree, node_address = nil, parent = nil, is_leaf = true,
                    prev_sibling = nil, next_sibling = nil,
-                   keys = [], values = [], children = [])
+                   keys = nil, values = nil, children = nil)
       @tree = tree
       if node_address == 0
         PEROBS.log.fatal "Node address may not be 0"
@@ -68,17 +68,17 @@ module PEROBS
       @parent = link(parent)
       @prev_sibling = link(prev_sibling)
       @next_sibling = link(next_sibling)
-      @keys = keys
+      @keys = keys || []
       if (@is_leaf = is_leaf)
-        @values = values
-        @children = []
+        @values = values || []
+        @children = nil
       else
-        @children = children
-        @values = []
+        @children = children || []
+        @values = nil
       end
     end
 
-    # Create a new SpaceTreeNode. This method should be used for the creation
+    # Create a new BTreeNode. This method should be used for the creation
     # of new nodes instead of calling the constructor directly.
     # @param tree [BTree] The tree the new node should belong to
     # @param parent [BTreeNode] The parent node
@@ -130,7 +130,8 @@ module PEROBS
       ary = bytes.unpack(BTreeNode::node_bytes_format(tree))
       # Read is_leaf
       if ary[0] != 0 && ary[0] != 1
-        PEROBS.log.fatal "First byte of a BTreeNode entry must be 0 or 1"
+        PEROBS.log.fatal "First byte of a BTreeNode entry at address " +
+          "#{address} must be 0 or 1 but is #{ary[0]}"
       end
       is_leaf = ary[0] == 0 ? false : true
       # This is the number of keys this node has.
@@ -350,6 +351,7 @@ module PEROBS
       unless @parent
         # The node is the root node. We need to create a parent node first.
         self.parent = link(BTreeNode::create(@tree, nil, false))
+        @tree.node_cache.insert(self)
         @parent.set_child(0, self)
         @tree.set_root(@parent)
       end
@@ -382,6 +384,7 @@ module PEROBS
       end
 
       i = search_key_index(key)
+      @tree.node_cache.insert(self)
       if @keys[i] == key
         # Overwrite existing entries
         @keys[i] = key
@@ -390,7 +393,6 @@ module PEROBS
         else
           @children[i + 1] = link(value_or_child)
         end
-        @tree.node_cache.insert(self)
 
         return false
       else
@@ -401,7 +403,6 @@ module PEROBS
         else
           @children.insert(i + 1, link(value_or_child))
         end
-        @tree.node_cache.insert(self)
 
         return true
       end
@@ -417,8 +418,8 @@ module PEROBS
       update_branch_key(key) if index == 0
 
       # Delete the corresponding value.
-      removed_value = @values.delete_at(index)
       @tree.node_cache.insert(self)
+      removed_value = @values.delete_at(index)
 
       if @keys.length < min_keys
         if @prev_sibling && @prev_sibling.parent == @parent
@@ -481,9 +482,9 @@ module PEROBS
         PEROBS.log.fatal "Leaf nodes are too big to merge"
       end
 
+      @tree.node_cache.insert(self)
       @keys += node.keys
       @values += node.values
-      @tree.node_cache.insert(self)
 
       node.parent.remove_child(node)
     end
@@ -494,11 +495,11 @@ module PEROBS
       end
 
       index = @parent.search_node_index(node) - 1
+      @tree.node_cache.insert(self)
       @keys << @parent.keys[index]
       @keys += node.keys
       node.children.each { |c| c.parent = link(self) }
       @children += node.children
-      @tree.node_cache.insert(self)
 
       node.parent.remove_child(node)
     end
@@ -525,6 +526,7 @@ module PEROBS
           "#{dest_node.is_leaf} node must be of same kind"
       end
 
+      @tree.node_cache.insert(dest_node)
       dest_node.keys[dst_idx, count] = @keys[src_idx, count]
       if @is_leaf
         # For leaves we copy the keys and corresponding values.
@@ -537,17 +539,17 @@ module PEROBS
           dest_node.set_child(dst_idx + i, @children[src_idx + i])
         end
       end
-      @tree.node_cache.insert(dest_node)
     end
 
     def parent=(p)
-      @parent = p
       @tree.node_cache.insert(self)
+      @parent = p
 
       p
     end
 
     def prev_sibling=(node)
+      @tree.node_cache.insert(self)
       @prev_sibling = node
       if node.nil? && @is_leaf
         # If this node is a leaf node without a previous sibling we need to
@@ -555,14 +557,12 @@ module PEROBS
         @tree.set_first_leaf(BTreeNodeLink.new(@tree, self))
       end
 
-      @tree.node_cache.insert(self)
-
       node
     end
 
     def next_sibling=(node)
-      @next_sibling = node
       @tree.node_cache.insert(self)
+      @next_sibling = node
       if node.nil? && @is_leaf
         # If this node is a leaf node without a next sibling we need to
         # register it as the last leaf node.
@@ -573,25 +573,25 @@ module PEROBS
     end
 
     def set_child(index, child)
+      @tree.node_cache.insert(self)
       if child
         @children[index] = link(child)
         @children[index].parent = link(self)
       else
         @children[index] = nil
       end
-      @tree.node_cache.insert(self)
 
       child
     end
 
     def trim(idx)
-      @keys = @keys[0..idx - 1]
+      @tree.node_cache.insert(self)
+      @keys.slice!(idx, @keys.length - idx)
       if @is_leaf
-        @values = @values[0..idx - 1]
+        @values.slice!(idx, @values.length - idx)
       else
-        @children = @children[0..idx]
+        @children.slice!(idx + 1, @children.length - idx - 1)
       end
-      @tree.node_cache.insert(self)
     end
 
     # Search the keys of the node that fits the given key. The result is
@@ -654,13 +654,18 @@ module PEROBS
     # @yield [key, value]
     # @return [nil or Hash] nil in case of errors or a hash with some
     #         statistical information about the tree
-    def check
+    def check(&block)
       stats = Stats.new(nil, 0, 0, 0)
 
       traverse do |node, position, stack|
         if position == 0
           stats.nodes_count += 1
           if node.parent
+            unless node.parent.is_a?(BTreeNodeLink)
+              node.error "parent is a #{node.parent.class} instead of a " +
+                "BTreeNodeLink"
+              return nil
+            end
             # After a split the nodes will only have half the maximum keys.
             # For branch nodes one of the split nodes will have even 1 key
             # less as this will become the branch key in a parent node.
@@ -695,6 +700,16 @@ module PEROBS
             else
               stats.branch_depth = node.tree_level
             end
+            if node.prev_sibling && !node.prev_sibling.is_a?(BTreeNodeLink)
+              node.error "prev_sibling is a #{node.prev_sibling.class} " +
+                "instead of a BTreeNodeLink"
+              return nil
+            end
+            if node.next_sibling && !node.next_sibling.is_a?(BTreeNodeLink)
+              node.error "next_sibling is a #{node.next_sibling.class} " +
+                "instead of a BTreeNodeLink"
+              return nil
+            end
             if node.prev_sibling.nil? && @tree.first_leaf != node
               node.error "Leaf node #{node.node_address} has no previous " +
                 "sibling but is not the first leaf of the tree"
@@ -708,9 +723,9 @@ module PEROBS
             unless node.keys.size == node.values.size
               node.error "Key count (#{node.keys.size}) and value " +
                 "count (#{node.values.size}) don't match"
-                return nil
+              return nil
             end
-            unless node.children.empty?
+            unless node.children.nil?
               node.error "@children must be nil for a leaf node"
               return nil
             end
@@ -718,14 +733,14 @@ module PEROBS
             stats.leave_nodes += 1
             stats.leaves += node.keys.length
           else
-            unless node.values.empty?
+            unless node.values.nil?
               node.error "@values must be nil for a branch node"
               return nil
             end
             unless node.children.size == node.keys.size + 1
               node.error "Key count (#{node.keys.size}) must be one " +
                 "less than children count (#{node.children.size})"
-                return nil
+              return nil
             end
             node.children.each_with_index do |child, i|
               unless child.is_a?(BTreeNodeLink)
@@ -789,7 +804,9 @@ module PEROBS
           else
             if block_given?
               # If a block was given, call this block with the key and value.
-              return nil unless yield(node.keys[index], node.values[index])
+              unless yield(node.keys[index], node.values[index])
+                return nil
+              end
             end
           end
         end

data/lib/perobs/BigArrayNode.rb

@@ -135,27 +135,29 @@ module PEROBS
       node = self
 
       # Traverse the tree to find the right node to add or replace the value.
+      idx = index
       while node do
         # Once we have reached a leaf node we can insert or replace the value.
         if node.is_leaf?
-          if index >= node.values.size
+          if idx >= node.values.size
             node.fatal "Set index (#{index}) larger than values array " +
-              "(#{node.values.size})."
+              "(#{idx} >= #{node.values.size})."
           end
-          node.values[index] = value
+          node.values[idx] = value
           return
         else
           # Descend into the right child node to add the value to.
-          cidx = node.search_child_index(index)
-          if (index -= node.offsets[cidx]) < 0
-            node.fatal "Index (#{index}) became negative"
+          cidx = node.search_child_index(idx)
+          if (idx -= node.offsets[cidx]) < 0
+            node.fatal "Idx (#{idx}) became negative while looking for " +
+              "index #{index}."
           end
           node = node.children[cidx]
         end
       end
 
       node.fatal "Could not find proper node to set the value while " +
-        "looking for index #{index}"
+        "looking for index #{index}."
     end
 
     # Insert the given value at the given index. All following values will be
@@ -610,7 +612,7 @@ module PEROBS
       end
     end
 
-    # @param index [offset] offset to search the child index for
+    # @param offset [Integer] offset to search the child index for
     # @return [Integer] Index of the matching offset or the insert position.
     def search_child_index(offset)
       # Handle special case for empty offsets list.
@@ -811,7 +813,7 @@ module PEROBS
 
     # Print and log an error message for the node.
     def fatal(msg)
-      msg = "Fatal error in BigArray node @#{@_id}: #{msg}\n" + @tree.to_s
+      msg = "Fatal error in BigArray node @#{@_id}: #{msg}\n"
       $stderr.puts msg
       PEROBS.log.fatal msg
     end

data/lib/perobs/Cache.rb

@@ -122,12 +122,25 @@ module PEROBS
     # @param id [Integer] ID of the cached PEROBS::ObjectBase
     def object_by_id(id)
       idx = id & @mask
-      # The index is just a hash. We still need to check if the object IDs are
-      # actually the same before we can return the object.
-      if (obj = @writes[idx]) && obj._id == id
-        # The object was in the write cache.
-        return obj
-      elsif (obj = @reads[idx]) && obj._id == id
+
+      if @transaction_stack.empty?
+        # The index is just a hash. We still need to check if the object IDs are
+        # actually the same before we can return the object.
+        if (obj = @writes[idx]) && obj._id == id
+          # The object was in the write cache.
+          return obj
+        end
+      else
+        # During transactions, the read cache is used to provide fast access
+        # to modified objects. But it does not store all modified objects
+        # since there can be hash collisions. So we also have to check all
+        # transaction objects first.
+        if (obj = @transaction_objects[id])
+          return obj
+        end
+      end
+
+      if (obj = @reads[idx]) && obj._id == id
         # The object was in the read cache.
         return obj
       end
@@ -152,10 +165,20 @@ module PEROBS
     # active, the write cache is flushed before the transaction is started.
     def begin_transaction
       if @transaction_stack.empty?
+        if @transaction_thread
+          PEROBS.log.fatal 'transaction_thread must be nil'
+        end
+        @transaction_thread = Thread.current
         # The new transaction is the top-level transaction. Flush the write
         # buffer to save the current state of all objects.
         flush
       else
+        # Nested transactions are currently only supported within the same
+        # thread. If we are in another thread, raise TransactionInOtherThread
+        # to pause the calling thread for a bit.
+        if @transaction_thread != Thread.current
+          raise TransactionInOtherThread
+        end
         # Save a copy of all objects that were modified during the enclosing
         # transaction.
         @transaction_stack.last.each do |id|
@@ -179,6 +202,7 @@ module PEROBS
         # into the backend storage.
         @transaction_stack.pop.each { |id| @transaction_objects[id]._sync }
         @transaction_objects = ::Hash.new
+        @transaction_thread = nil
       else
         # A nested transaction completed successfully. We add the list of
         # modified objects to the list of the enclosing transaction.
@@ -200,6 +224,7 @@ module PEROBS
       @transaction_stack.pop.each do |id|
         @transaction_objects[id]._restore(@transaction_stack.length)
       end
+      @transaction_thread = nil
     end
 
     # Clear all cached entries. You must call flush before calling this
@@ -211,6 +236,7 @@ module PEROBS
       @reads = ::Array.new(2 ** @bits)
       @writes = ::Array.new(2 ** @bits)
       @transaction_stack = ::Array.new
+      @transaction_thread = nil
       @transaction_objects = ::Hash.new
     end
 

data/lib/perobs/EquiBlobsFile.rb

@@ -425,6 +425,8 @@ module PEROBS
                    pack("Q#{@custom_data_values.length}"))
         end
         @f.flush
+      rescue IOError => e
+        PEROBS.log.fatal "Cannot write EquiBlobsFile header: " + e.message
       end
     end