perobs 4.0.0 → 4.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a988f009b1b97bc8da2b2bcfeb65399284a07032
-  data.tar.gz: 4346d5cdbec5b4154741f0a41d3f526985803912
+SHA256:
+  metadata.gz: c2c526c97aab15c09f8e8acbcb432203e7d60fef8c504df6ddb0b8f35459cca0
+  data.tar.gz: ad077bb879c041289c7a474e87645064ff84203c674640aff59b5f23923ac1b7
 SHA512:
-  metadata.gz: 639673c10faa5082c742258ee9848f5e951c6d1ec7fc9f03ec7066260ea5042b6c8b90020df6fa506b8be54cc5fd9793a0939f80ef1da726c27832c8a2eb8c85
-  data.tar.gz: 92e63145a1fc9b6e76dbdf6f4cdebfb5a8935bfcbd93a9634e22c12f874b945b656a17fe93aa03a7a3189a00371d2e2abd8d411862ad0ad51b0c364f7b7dc027
+  metadata.gz: b86fc662ef8bbf623ee7da777f023a74e3db9502df18c45dcda9ce244982225aaec7e74ce25378147c807775765c57d801810ac16be01d2145227bb0642caa3b
+  data.tar.gz: d0c5a79eb60a1221bc385a1fafbc77a1aaa1e47782b3ea27ca92f5f7c539804bc00a388fdddc1ab956d7805dbdd52d055f5cdd76a8bb155ec5ada4335b7b9175

data/README.md

@@ -6,11 +6,12 @@ them from PEROBS::Object. They will be in memory when needed and
 transparently stored into a persistent storage.
 
 This library is ideal for Ruby applications that work on huge, mostly
-constant data sets and usually handle a small subset of the data at a
+static data sets and usually process a small subset of the data at a
 time. To ensure data consistency of a larger data set, you can use
 transactions to make modifications of multiple objects atomicaly.
 Transactions can be nested and are aborted when an exception is
-raised.
+raised. PEROBS is thread-safe, so you can use it in a multi-threaded
+application.
 
 ## Usage
 
@@ -108,7 +109,7 @@ class Person < PEROBS::Object
     attr_init(:father) do { @store.new(Person, 'Dad') }
   end
 
-  def merry(spouse)
+  def marry(spouse)
     self.spouse = spouse
     self.status = :married
   end
@@ -120,15 +121,18 @@ class Person < PEROBS::Object
 
 end
 
-store = PEROBS::Store.new('family')
-store['grandpa'] = joe = store.new(Person, 'Joe')
-store['grandma'] = jane = store.new(Person, 'Jane')
-jim = store.new(Person, 'Jim')
-jim.father = joe
-joe.kids << jim
-jim.mother = jane
-jane.kids << jim
-store.exit
+begin
+  store = PEROBS::Store.new('family')
+  store['grandpa'] = joe = store.new(Person, 'Joe')
+  store['grandma'] = jane = store.new(Person, 'Jane')
+  jim = store.new(Person, 'Jim')
+  jim.father = joe
+  joe.kids << jim
+  jim.mother = jane
+  jane.kids << jim
+ensure
+  store.exit
+end
 ```
 
 When you run this script, a folder named 'family' will be created. It
@@ -166,9 +170,15 @@ object to another object.
 
 ### Caveats and known issues
 
-PEROBS is currently not thread-safe. You cannot simultaneously access
-the database from multiple application. The library uses locks to
-ensure that only one Store object is accessing the database at a time.
+You cannot simultaneously access the database from multiple
+applications concurrently. The library uses locks to ensure that only
+one Store object is accessing the database at a time.
+
+In case the application terminates without calling Store::exit(), the
+database or the database index could get corrupted. To check the
+consistency of your database you can use Store::check(). To check and
+repair the database you can call Store::repair(). Depending on the
+size of your database, these operations can last minutes to hours.
 
 ## Installation
 
@@ -188,7 +198,8 @@ Or install it yourself as:
 
 ## Copyright and License
 
-Copyright (c) 2015, 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+Copyright (c) 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 by
+Chris Schlaeger <chris@taskjuggler.org>
 
 PEROBS and all accompanying files are licensed under this MIT License
 

data/lib/perobs/Array.rb

@@ -2,7 +2,7 @@
 #
 # = Array.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -44,19 +44,27 @@ module PEROBS
 
     attr_reader :data
 
-    # These methods do not mutate the Array. They only perform read
-    # operations.
+    # These methods do not mutate the Array but create a new PEROBS::Array
+    # object. They only perform read operations.
     ([
-      :&, :*, :+, :-, :==, :[], :<=>, :at, :abbrev, :assoc, :bsearch, :collect,
-      :combination, :compact, :count, :cycle, :dclone, :drop, :drop_while,
+      :|, :&, :+, :-, :collect, :compact, :drop, :drop_while,
+      :flatten, :map, :reject, :reverse, :rotate, :select, :shuffle, :slice,
+      :sort, :take, :take_while, :uniq, :values_at
+    ] + Enumerable.instance_methods).uniq.each do |method_sym|
+      define_method(method_sym) do |*args, &block|
+        @store.cache.cache_read(self)
+        @store.new(PEROBS::Array, @data.send(method_sym, *args, &block))
+      end
+    end
+
+    # These methods do not mutate the Array and only perform read operations.
+    # They do not return basic objects types.
+    ([
+      :==, :[], :<=>, :at, :bsearch, :bsearch_index, :count, :cycle,
       :each, :each_index, :empty?, :eql?, :fetch, :find_index, :first,
-      :flatten, :frozen?, :hash, :include?, :index, :join, :last,
-      :length, :map, :pack, :permutation, :pretty_print, :pretty_print_cycle,
-      :product, :rassoc, :reject, :repeated_combination,
-      :repeated_permutation, :reverse, :reverse_each, :rindex, :rotate,
-      :sample, :select, :shelljoin, :shuffle, :size, :slice, :sort, :take,
-      :take_while, :to_a, :to_ary, :to_s, :transpose, :uniq, :values_at, :zip,
-      :|
+      :frozen?, :include?, :index, :join, :last, :length, :pack,
+      :pretty_print, :pretty_print_cycle, :reverse_each, :rindex, :sample,
+      :size, :to_a, :to_ary, :to_s
     ] + Enumerable.instance_methods).uniq.each do |method_sym|
       define_method(method_sym) do |*args, &block|
         @store.cache.cache_read(self)
@@ -64,12 +72,23 @@ module PEROBS
       end
     end
 
+    # These methods mutate the Array and return self.
+    [
+      :<<, :clear, :collect!, :compact!, :concat,
+      :fill, :flatten!, :insert, :keep_if, :map!, :push,
+      :reject!, :replace, :select!, :reverse!, :rotate!, :shuffle!,
+      :slice!, :sort!, :sort_by!, :uniq!
+    ].each do |method_sym|
+      define_method(method_sym) do |*args, &block|
+        @store.cache.cache_write(self)
+        @data.send(method_sym, *args, &block)
+        myself
+      end
+    end
+
     # These methods mutate the Array.
     [
-      :<<, :[]=, :clear, :collect!, :compact!, :concat, :delete, :delete_at,
-      :delete_if, :fill, :flatten!, :insert, :keep_if, :map!, :pop, :push,
-      :reject!, :replace, :select!, :reverse!, :rotate!, :shift, :shuffle!,
-      :slice!, :sort!, :sort_by!, :uniq!, :unshift
+      :delete, :delete_at, :delete_if, :shift, :pop
     ].each do |method_sym|
       define_method(method_sym) do |*args, &block|
         @store.cache.cache_write(self)
@@ -81,17 +100,45 @@ module PEROBS
     # PEROBS users should never call this method or equivalents of derived
     # methods directly.
     # @param p [PEROBS::Handle] PEROBS handle
-    # @param size [Integer] The requested size of the Array
+    # @param arg1 [Integer or Array] The requested size of the Array or an
+    #        Array to initialize
     # @param default [Any] The default value that is returned when no value is
     #        stored for a specific key.
-    def initialize(p, size = 0, default = nil)
+    def initialize(p, arg1 = 0, default = nil, &block)
       super(p)
-      @data = ::Array.new(size, default)
+      if arg1.is_a?(::Array)
+        arg1.each { |v| _check_assignment_value(v) }
+        @data = arg1.dup
+      elsif block_given?
+        @data = ::Array.new(arg1) do
+          _check_assignment_value(yield)
+        end
+      else
+        @data = ::Array.new(arg1, _check_assignment_value(default))
+      end
 
       # Ensure that the newly created object will be pushed into the database.
       @store.cache.cache_write(self)
     end
 
+    # Proxy for the assignment method.
+    def []=(*args)
+      if (args.length == 2)
+        _check_assignment_value(args[1])
+      else
+        _check_assignment_value(args[2])
+      end
+      @store.cache.cache_write(self)
+      @data.[]=(*args)
+    end
+
+    # Proxy for the unshift method.
+    def unshift(val)
+      _check_assignment_value(val)
+      @store.cache.cache_write(self)
+      @data.unshift(val)
+    end
+
     # Return a list of all object IDs of all persistend objects that this Array
     # is referencing.
     # @return [Array of Integer] IDs of referenced objects

data/lib/perobs/BTree.rb

@@ -2,7 +2,7 @@
 #
 # = BTreeNode.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2016, 2017, 2018 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -32,7 +32,7 @@ require 'perobs/BTreeNode'
 
 module PEROBS
 
-  # This BTree class is very similar to a classic BTree implementation. It
+  # This BTree class is very similar to a classic B+Tree implementation. It
   # manages a tree that is always balanced. The BTree is stored in the
   # specified directory and partially kept in memory to speed up operations.
   # The order of the tree specifies how many keys each node will be able to
@@ -40,18 +40,21 @@ module PEROBS
   # have N + 1 references to child nodes instead.
   class BTree
 
-    attr_reader :order, :nodes, :node_cache
+    attr_reader :order, :nodes, :node_cache, :first_leaf, :last_leaf, :size
 
     # Create a new BTree object.
     # @param dir [String] Directory to store the tree file
     # @param name [String] Base name of the BTree related files in 'dir'
     # @param order [Integer] The maximum number of keys per node. This number
     #        must be odd and larger than 2 and smaller than 2**16 - 1.
-    def initialize(dir, name, order)
+    # @param progressmeter [ProgressMeter] reference to a ProgressMeter object
+    def initialize(dir, name, order, progressmeter)
       @dir = dir
       @name = name
-      unless order > 2
-        PEROBS.log.fatal "BTree order must be larger than 2, not #{order}"
+      @progressmeter = progressmeter
+
+      unless order > 4
+        PEROBS.log.fatal "BTree order must be larger than 4, not #{order}"
       end
       unless order % 2 == 1
         PEROBS.log.fatal "BTree order must be an uneven number, not #{order}"
@@ -62,9 +65,14 @@ module PEROBS
       @order = order
 
       # This EquiBlobsFile contains the nodes of the BTree.
-      @nodes = EquiBlobsFile.new(@dir, @name,
+      @nodes = EquiBlobsFile.new(@dir, @name, @progressmeter,
                                  BTreeNode::node_bytes(@order))
-      @node_cache = PersistentObjectCache.new(512, BTreeNode, self)
+      @nodes.register_custom_data('first_leaf')
+      @nodes.register_custom_data('last_leaf')
+      @nodes.register_custom_data('btree_size')
+      @node_cache = PersistentObjectCache.new(2**13, 2**13, BTreeNode, self)
+      @root = @first_leaf = @last_leaf = nil
+      @size = 0
 
       # This BTree implementation uses a write cache to improve write
       # performance of multiple successive read/write operations. This also
@@ -89,23 +97,44 @@ module PEROBS
 
       @node_cache.clear
       @nodes.open
-      node = @nodes.total_entries == 0 ?
-        BTreeNode::create(self) :
-        BTreeNode::load(self, @nodes.first_entry)
+
+      if @nodes.total_entries == 0
+        # We've created a new nodes file
+        node = BTreeNode::create(self)
+      else
+        # We are loading an existing tree.
+        node = BTreeNode::load_and_link(self, @nodes.first_entry)
+        @first_leaf = BTreeNode::load_and_link(
+          self, @nodes.get_custom_data('first_leaf'))
+        @last_leaf = BTreeNode::load_and_link(
+          self, @nodes.get_custom_data('last_leaf'))
+      end
       set_root(node)
+
+      # Get the total number of entries that are stored in the tree.
+      @size = @nodes.get_custom_data('btree_size')
     end
 
     # Close the tree file.
     def close
       sync
+      PEROBS.log.info "BTree file #{@name} has currently " +
+        "#{@nodes.total_entries} used entries and #{@nodes.total_spaces} " +
+        "unused entries"
       @nodes.close
       @root = nil
     end
 
+    # @return true if file is currently open
+    def is_open?
+      !@root.nil?
+    end
+
     # Clear all pools and forget any registered spaces.
     def clear
       @node_cache.clear
       @nodes.clear
+      @size = 0
       set_root(BTreeNode::create(self))
     end
 
@@ -114,6 +143,7 @@ module PEROBS
     # all stored data from the BTree.
     def erase
       @nodes.erase
+      @size = 0
       @root = nil
       @dirty_flag.forced_unlock
     end
@@ -121,6 +151,7 @@ module PEROBS
     # Flush all pending modifications into the tree file.
     def sync
       @node_cache.flush(true)
+      @nodes.set_custom_data('btree_size', @size)
       @nodes.sync
       @dirty_flag.unlock if @dirty_flag.is_locked?
     end
@@ -128,22 +159,70 @@ module PEROBS
     # Check if the tree file contains any errors.
     # @return [Boolean] true if no erros were found, false otherwise
     def check(&block)
-      @root.check(&block)
+      sync
+      return false unless @nodes.check
+
+      entries = 0
+      stats = nil
+      @progressmeter.start('Checking index structure', @size) do |pm|
+        stats = @root.check do |k, v|
+          pm.update(entries += 1)
+          block_given? ? yield(k, v) : true
+        end
+      end
+
+      return false unless stats
+
+      unless entries == @size
+        PEROBS.log.error "The BTree size (#{@size}) and the number of " +
+          "found entries (#{entries}) don't match"
+        return false
+      end
+      unless stats.nodes_count == @nodes.total_entries
+        PEROBS.log.error "The BTree nodes count (#{stats.nodes_count}) and " +
+          "the number of entries in the nodes file (#{@nodes.total_entries}) " +
+          "don't match"
+          return false
+      end
+      PEROBS.log.info "Statistics for the BTree #{@name}: " +
+        "Number of nodes: #{stats.nodes_count}; " +
+        "Branch depth: #{stats.branch_depth}; " +
+        "Number of leave nodes: #{stats.leave_nodes}; " +
+        "Number of leaves: #{stats.leaves}"
+
+      true
     end
 
     # Register a new node as root node of the tree.
+    # @param node [BTreeNode]
     def set_root(node)
       @root = node
       @nodes.first_entry = node.node_address
     end
 
+    # Set the address of the first leaf node.
+    # @param node [BTreeNode]
+    def set_first_leaf(node)
+      @first_leaf = node
+      @nodes.set_custom_data('first_leaf', node.node_address)
+    end
+
+    # Set the address of the last leaf node.
+    # @param node [BTreeNode]
+    def set_last_leaf(node)
+      @last_leaf = node
+      @nodes.set_custom_data('last_leaf', node.node_address)
+    end
+
     # Insert a new value into the tree using the key as a unique index. If the
     # key already exists the old value will be overwritten.
     # @param key [Integer] Unique key
     # @param value [Integer] value
     def insert(key, value)
-      @root.insert(key, value)
-      @node_cache.flush
+      if @root.insert(key, value)
+        @size += 1
+        @node_cache.flush
+      end
     end
 
     # Retrieve the value associated with the given key. If no entry was found,
@@ -154,10 +233,17 @@ module PEROBS
       @root.get(key)
     end
 
+    # Either return the key/value pair that exactly matches the key or a
+    # key/value pair that has a key that is at least min_miss_increment larger
+    # than the key.
+    def get_best_match(key, min_miss_increment)
+      @root.get_best_match(key, min_miss_increment)
+    end
+
     # Find and remove the value associated with the given key. If no entry was
     # found, return nil, otherwise the found value.
     def remove(key)
-      removed_value = @root.remove(key)
+      @size -= 1 unless (removed_value = @root.remove(key)).nil?
 
       # Check if the root node only contains one child link after the delete
       # operation. Then we can delete that node and pull the tree one level
@@ -187,6 +273,11 @@ module PEROBS
       @nodes.delete_blob(address)
     end
 
+    # @return [Integer] The number of entries stored in the tree.
+    def entries_count
+      @size
+    end
+
     # @return [String] Human reable form of the tree.
     def to_s
       @root.to_s

data/lib/perobs/BTreeBlob.rb

@@ -2,7 +2,7 @@
 #
 # = BTreeBlob.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2019 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -33,7 +33,7 @@ require 'perobs/RobustFile'
 module PEROBS
 
   # This class manages the usage of the data blobs in the corresponding
-  # HashedBlobsDB object.
+  # BTreeDB object.
   class BTreeBlob
 
     # Magic number used for index files.
@@ -144,11 +144,12 @@ module PEROBS
 
     # Remove all entries from the index that have not been marked.
     # @return [Array] List of deleted object IDs.
-    def delete_unmarked_entries
+    def delete_unmarked_entries(&block)
       deleted_ids = []
       # First remove the entry from the hash table.
       @entries_by_id.delete_if do |id, e|
         if e[MARKED] == 0
+          yield(id) if block_given?
           deleted_ids << id
           true
         else

data/lib/perobs/BTreeDB.rb

@@ -2,7 +2,8 @@
 #
 # = BTreeDB.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2018, 2019
+# by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -58,7 +59,7 @@ module PEROBS
     #                         nodes. The insert/find/delete time grows
     #                         linearly with the size.
     def initialize(db_name, options = {})
-      super(options[:serializer] || :json)
+      super(options)
 
       @db_dir = db_name
       # Create the database directory if it doesn't exist yet.
@@ -159,9 +160,9 @@ module PEROBS
     # Permanently delete all objects that have not been marked. Those are
     # orphaned and are no longer referenced by any actively used object.
     # @return [Array] List of IDs that have been removed from the DB.
-    def delete_unmarked_objects
+    def delete_unmarked_objects(&block)
       deleted_ids = []
-      each_blob { |blob| deleted_ids += blob.delete_unmarked_entries }
+      each_blob { |blob| deleted_ids += blob.delete_unmarked_entries(&block) }
       deleted_ids
     end