perobs 3.0.1 → 4.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 20f164ff526b984530355e99ab38623fd2b87cea
-  data.tar.gz: befc7531c92e0dc86c23b058cbc572657f66661f
+SHA256:
+  metadata.gz: a61fc945e0ef9f5ed6558080931d2acae42cc0401f375275684e4ee32fefe4f7
+  data.tar.gz: 4d864fdc0791aa78d8c180b4686ee825cd25e209284fca1966f144813c063280
 SHA512:
-  metadata.gz: 3b18e2e16719f896d8500852f0c1529ce2a1d9816695df04be2203cd1a10da8b70fe737afa910ec1fed71855eb6534da2b0dd16f8c6428c0ea1ae06a6509b59f
-  data.tar.gz: d6de8c99f6cffcb245b5468c6f6b5f4b5572c44019f7b9926adeae126a81ad507b815bbefda24b07f4724bc0a11abc0ab3235b304d2ca7180cde3bdc628c0b1e
+  metadata.gz: f3834a9caae693d82837fb9f75141cb35e85f1a2c1439d1bb898f8578d9ae082f46deb0233d2b8a02d6ae6b0bf66862098b47ff571ff3d9a6b874fadaef6d23a
+  data.tar.gz: 883f1b5e553fae2be0039aa090d89bf6eb44ec1d0dc31488aeaa727ec8bc2844c9b722568cd7dde0b33c507121afd11e720154d4ff27e66aa3ac3812d5603954

data/README.md

@@ -30,7 +30,7 @@ classes:
 
 When you derive your own class from PEROBS::Object you need to
 specify which instance variables should be persistent. By using
-po_attr you can provide a list of symbols that describe the instance
+'attr_persist' you can provide a list of symbols that declare the instance
 variables to persist. This will also create getter and setter methods
 for these instance varables.  You can set default values in the
 constructor . The constructor of PEROBS::ObjectBase derived objects
@@ -61,13 +61,14 @@ almost every Ruby data type. YAML is much slower than JSON and Marshal
 is not guaranteed to be compatible between Ruby versions.
 
 Once you have created a store you can assign objects to it. All
-persistent objects must be created with Store.new(). This is
-necessary as you will only deal with proxy objects in your code.
-Except for the member methods, you will never deal with the objects
-directly. Instead Store.new() returns a POXReference object that acts
-as a transparent proxy. This proxy is needed as your code never knows
-if the actual object is really loaded into the memory or not. PEROBS
-will handle this transparently for you.
+persistent objects must be created with Store.new(). The Store object
+is available via the @store instance variable provided by the parent
+class. This is necessary as you will only deal with proxy objects in
+your code.  Except for the member methods, you will never deal with
+the objects directly. Instead Store.new() returns a POXReference
+object that acts as a transparent proxy. This proxy is needed as your
+code never knows if the actual object is really loaded into the memory
+or not. PEROBS will handle this transparently for you.
 
 A build-in cache keeps access latencies to recently used objects low
 and lazily flushes modified objects into the persistend back-end when
@@ -92,22 +93,22 @@ require 'perobs'
 
 class Person < PEROBS::Object
 
-  po_attr :name, :mother, :father, :kids, :spouse, :status
+  attr_persist :name, :mother, :father, :kids, :spouse, :status
 
   def initialize(p, name)
     super(p)
-    attr_init(:name, name)
-    attr_init(:kids, store.new(PEROBS::Array))
-    attr_init(:status, :single)
+    self.name = name
+    self.kids = @store.new(PEROBS::Array)
+    self.status = :single
   end
 
   def restore
     # Use block version of attr_init() to avoid creating unneded
     # objects. The block is only called when @father doesn't exist yet.
-    attr_init(:father) do { store.new(Person, 'Dad') }
+    attr_init(:father) do { @store.new(Person, 'Dad') }
   end
 
-  def merry(spouse)
+  def marry(spouse)
     self.spouse = spouse
     self.status = :married
   end
@@ -136,7 +137,7 @@ contains the 3 Person objects.
 ### Accessing persistent instance variables
 
 All instance variables that should be persisted must be declared with
-'po_attr'. This will create the instance variable, a getter and setter
+'attr_persist'. This will create the instance variable, a getter and setter
 method for it. These getter and setter methods are the recommended way
 to access instance variables both from ouside of the instances as well
 as from within. To access the setter or getter method from within an
@@ -166,8 +167,8 @@ object to another object.
 ### Caveats and known issues
 
 PEROBS is currently not thread-safe. You cannot simultaneously access
-the database from multiple application. You must provide your own
-locking mechanism to prevent this from happening.
+the database from multiple application. The library uses locks to
+ensure that only one Store object is accessing the database at a time.
 
 ## Installation
 
@@ -187,7 +188,7 @@ Or install it yourself as:
 
 ## Copyright and License
 
-Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+Copyright (c) 2015, 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
 
 PEROBS and all accompanying files are licensed under this MIT License
 

data/lib/perobs.rb

@@ -27,3 +27,5 @@
 
 require "perobs/version"
 require 'perobs/Store'
+require 'perobs/ConsoleProgressMeter'
+require 'perobs/FuzzyStringMatcher'

data/lib/perobs/Array.rb

@@ -2,7 +2,7 @@
 #
 # = Array.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2015, 2016 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2015, 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -44,19 +44,27 @@ module PEROBS
 
     attr_reader :data
 
-    # These methods do not mutate the Array. They only perform read
-    # operations.
+    # These methods do not mutate the Array but create a new PEROBS::Array
+    # object. They only perform read operations.
     ([
-      :&, :*, :+, :-, :==, :[], :<=>, :at, :abbrev, :assoc, :bsearch, :collect,
-      :combination, :compact, :count, :cycle, :dclone, :drop, :drop_while,
+      :|, :&, :+, :-, :collect, :compact, :drop, :drop_while,
+      :flatten, :map, :reject, :reverse, :rotate, :select, :shuffle, :slice,
+      :sort, :take, :take_while, :uniq, :values_at
+    ] + Enumerable.instance_methods).uniq.each do |method_sym|
+      define_method(method_sym) do |*args, &block|
+        @store.cache.cache_read(self)
+        @store.new(PEROBS::Array, @data.send(method_sym, *args, &block))
+      end
+    end
+
+    # These methods do not mutate the Array and only perform read operations.
+    # They do not return basic objects types.
+    ([
+      :==, :[], :<=>, :at, :bsearch, :bsearch_index, :count, :cycle,
       :each, :each_index, :empty?, :eql?, :fetch, :find_index, :first,
-      :flatten, :frozen?, :hash, :include?, :index, :join, :last,
-      :length, :map, :pack, :permutation, :pretty_print, :pretty_print_cycle,
-      :product, :rassoc, :reject, :repeated_combination,
-      :repeated_permutation, :reverse, :reverse_each, :rindex, :rotate,
-      :sample, :select, :shelljoin, :shuffle, :size, :slice, :sort, :take,
-      :take_while, :to_a, :to_ary, :to_s, :transpose, :uniq, :values_at, :zip,
-      :|
+      :frozen?, :include?, :index, :join, :last, :length, :pack,
+      :pretty_print, :pretty_print_cycle, :reverse_each, :rindex, :sample,
+      :size, :to_a, :to_ary, :to_s
     ] + Enumerable.instance_methods).uniq.each do |method_sym|
       define_method(method_sym) do |*args, &block|
         @store.cache.cache_read(self)
@@ -64,12 +72,23 @@ module PEROBS
       end
     end
 
+    # These methods mutate the Array and return self.
+    [
+      :<<, :clear, :collect!, :compact!, :concat,
+      :fill, :flatten!, :insert, :keep_if, :map!, :push,
+      :reject!, :replace, :select!, :reverse!, :rotate!, :shuffle!,
+      :slice!, :sort!, :sort_by!, :uniq!
+    ].each do |method_sym|
+      define_method(method_sym) do |*args, &block|
+        @store.cache.cache_write(self)
+        @data.send(method_sym, *args, &block)
+        myself
+      end
+    end
+
     # These methods mutate the Array.
     [
-      :<<, :[]=, :clear, :collect!, :compact!, :concat, :delete, :delete_at,
-      :delete_if, :fill, :flatten!, :insert, :keep_if, :map!, :pop, :push,
-      :reject!, :replace, :select!, :reverse!, :rotate!, :shift, :shuffle!,
-      :slice!, :sort!, :sort_by!, :uniq!, :unshift
+      :delete, :delete_at, :delete_if, :shift, :pop
     ].each do |method_sym|
       define_method(method_sym) do |*args, &block|
         @store.cache.cache_write(self)
@@ -81,20 +100,48 @@ module PEROBS
     # PEROBS users should never call this method or equivalents of derived
     # methods directly.
     # @param p [PEROBS::Handle] PEROBS handle
-    # @param size [Fixnum] The requested size of the Array
+    # @param arg1 [Integer or Array] The requested size of the Array or an
+    #        Array to initialize
     # @param default [Any] The default value that is returned when no value is
     #        stored for a specific key.
-    def initialize(p, size = 0, default = nil)
+    def initialize(p, arg1 = 0, default = nil, &block)
       super(p)
-      @data = ::Array.new(size, default)
+      if arg1.is_a?(::Array)
+        arg1.each { |v| _check_assignment_value(v) }
+        @data = arg1.dup
+      elsif block_given?
+        @data = ::Array.new(arg1) do
+          _check_assignment_value(yield)
+        end
+      else
+        @data = ::Array.new(arg1, _check_assignment_value(default))
+      end
 
       # Ensure that the newly created object will be pushed into the database.
       @store.cache.cache_write(self)
     end
 
+    # Proxy for the assignment method.
+    def []=(*args)
+      if (args.length == 2)
+        _check_assignment_value(args[1])
+      else
+        _check_assignment_value(args[2])
+      end
+      @store.cache.cache_write(self)
+      @data.[]=(*args)
+    end
+
+    # Proxy for the unshift method.
+    def unshift(val)
+      _check_assignment_value(val)
+      @store.cache.cache_write(self)
+      @data.unshift(val)
+    end
+
     # Return a list of all object IDs of all persistend objects that this Array
     # is referencing.
-    # @return [Array of Fixnum or Bignum] IDs of referenced objects
+    # @return [Array of Integer] IDs of referenced objects
     def _referenced_object_ids
       @data.each.select do |v|
         v && v.respond_to?(:is_poxreference?)
@@ -103,7 +150,7 @@ module PEROBS
 
     # This method should only be used during store repair operations. It will
     # delete all references to the given object ID.
-    # @param id [Fixnum/Bignum] targeted object ID
+    # @param id [Integer] targeted object ID
     def _delete_reference_to_id(id)
       @data.delete_if do |v|
         v && v.respond_to?(:is_poxreference?) && v.id == id

data/lib/perobs/BTree.rb

@@ -2,7 +2,7 @@
 #
 # = BTreeNode.rb -- Persistent Ruby Object Store
 #
-# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+# Copyright (c) 2016, 2017, 2018 by Chris Schlaeger <chris@taskjuggler.org>
 #
 # MIT License
 #
@@ -27,12 +27,12 @@
 
 require 'perobs/LockFile'
 require 'perobs/EquiBlobsFile'
-require 'perobs/BTreeNodeCache'
+require 'perobs/PersistentObjectCache'
 require 'perobs/BTreeNode'
 
 module PEROBS
 
-  # This BTree class is very similar to a classic BTree implementation. It
+  # This BTree class is very similar to a classic B+Tree implementation. It
   # manages a tree that is always balanced. The BTree is stored in the
   # specified directory and partially kept in memory to speed up operations.
   # The order of the tree specifies how many keys each node will be able to
@@ -40,18 +40,21 @@ module PEROBS
   # have N + 1 references to child nodes instead.
   class BTree
 
-    attr_reader :order, :nodes
+    attr_reader :order, :nodes, :node_cache, :first_leaf, :last_leaf, :size
 
     # Create a new BTree object.
     # @param dir [String] Directory to store the tree file
     # @param name [String] Base name of the BTree related files in 'dir'
     # @param order [Integer] The maximum number of keys per node. This number
     #        must be odd and larger than 2 and smaller than 2**16 - 1.
-    def initialize(dir, name, order)
+    # @param progressmeter [ProgressMeter] reference to a ProgressMeter object
+    def initialize(dir, name, order, progressmeter)
       @dir = dir
       @name = name
-      unless order > 2
-        PEROBS.log.fatal "BTree order must be larger than 2, not #{order}"
+      @progressmeter = progressmeter
+
+      unless order > 4
+        PEROBS.log.fatal "BTree order must be larger than 4, not #{order}"
       end
       unless order % 2 == 1
         PEROBS.log.fatal "BTree order must be an uneven number, not #{order}"
@@ -62,9 +65,14 @@ module PEROBS
       @order = order
 
       # This EquiBlobsFile contains the nodes of the BTree.
-      @nodes = EquiBlobsFile.new(@dir, @name,
+      @nodes = EquiBlobsFile.new(@dir, @name, @progressmeter,
                                  BTreeNode::node_bytes(@order))
-      @node_cache = BTreeNodeCache.new
+      @nodes.register_custom_data('first_leaf')
+      @nodes.register_custom_data('last_leaf')
+      @nodes.register_custom_data('btree_size')
+      @node_cache = PersistentObjectCache.new(2**13, 2**13, BTreeNode, self)
+      @root = @first_leaf = @last_leaf = nil
+      @size = 0
 
       # This BTree implementation uses a write cache to improve write
       # performance of multiple successive read/write operations. This also
@@ -89,22 +97,45 @@ module PEROBS
 
       @node_cache.clear
       @nodes.open
-      set_root(new_node(nil, @nodes.total_entries == 0 ?
-                             nil : @nodes.first_entry))
+
+      if @nodes.total_entries == 0
+        # We've created a new nodes file
+        node = BTreeNode::create(self)
+      else
+        # We are loading an existing tree.
+        node = BTreeNode::load_and_link(self, @nodes.first_entry)
+        @first_leaf = BTreeNode::load_and_link(
+          self, @nodes.get_custom_data('first_leaf'))
+        @last_leaf = BTreeNode::load_and_link(
+          self, @nodes.get_custom_data('last_leaf'))
+      end
+      set_root(node)
+
+      # Get the total number of entries that are stored in the tree.
+      @size = @nodes.get_custom_data('btree_size')
     end
 
     # Close the tree file.
     def close
       sync
+      PEROBS.log.info "BTree file #{@name} has currently " +
+        "#{@nodes.total_entries} used entries and #{@nodes.total_spaces} " +
+        "unused entries"
       @nodes.close
       @root = nil
     end
 
+    # @return true if file is currently open
+    def is_open?
+      !@root.nil?
+    end
+
     # Clear all pools and forget any registered spaces.
     def clear
       @node_cache.clear
       @nodes.clear
-      set_root(new_node(nil))
+      @size = 0
+      set_root(BTreeNode::create(self))
     end
 
     # Erase the backing store of the BTree. This method should only be called
@@ -112,12 +143,15 @@ module PEROBS
     # all stored data from the BTree.
     def erase
       @nodes.erase
+      @size = 0
+      @root = nil
       @dirty_flag.forced_unlock
     end
 
     # Flush all pending modifications into the tree file.
     def sync
       @node_cache.flush(true)
+      @nodes.set_custom_data('btree_size', @size)
       @nodes.sync
       @dirty_flag.unlock if @dirty_flag.is_locked?
     end
@@ -125,14 +159,59 @@ module PEROBS
     # Check if the tree file contains any errors.
     # @return [Boolean] true if no erros were found, false otherwise
     def check(&block)
-      @root.check(&block)
+      sync
+      return false unless @nodes.check
+
+      entries = 0
+      stats = nil
+      @progressmeter.start('Checking index structure', @size) do |pm|
+        stats = @root.check do |k, v|
+          pm.update(entries += 1)
+          block_given? ? yield(k, v) : true
+        end
+      end
+
+      return false unless stats
+
+      unless entries == @size
+        PEROBS.log.error "The BTree size (#{@size}) and the number of " +
+          "found entries (#{entries}) don't match"
+        return false
+      end
+      unless stats.nodes_count == @nodes.total_entries
+        PEROBS.log.error "The BTree nodes count (#{stats.nodes_count}) and " +
+          "the number of entries in the nodes file (#{@nodes.total_entries}) " +
+          "don't match"
+          return false
+      end
+      PEROBS.log.info "Statistics for the BTree #{@name}: " +
+        "Number of nodes: #{stats.nodes_count}; " +
+        "Branch depth: #{stats.branch_depth}; " +
+        "Number of leave nodes: #{stats.leave_nodes}; " +
+        "Number of leaves: #{stats.leaves}"
+
+      true
     end
 
     # Register a new node as root node of the tree.
+    # @param node [BTreeNode]
     def set_root(node)
       @root = node
       @nodes.first_entry = node.node_address
-      @node_cache.set_root(node)
+    end
+
+    # Set the address of the first leaf node.
+    # @param node [BTreeNode]
+    def set_first_leaf(node)
+      @first_leaf = node
+      @nodes.set_custom_data('first_leaf', node.node_address)
+    end
+
+    # Set the address of the last leaf node.
+    # @param node [BTreeNode]
+    def set_last_leaf(node)
+      @last_leaf = node
+      @nodes.set_custom_data('last_leaf', node.node_address)
     end
 
     # Insert a new value into the tree using the key as a unique index. If the
@@ -140,8 +219,10 @@ module PEROBS
     # @param key [Integer] Unique key
     # @param value [Integer] value
     def insert(key, value)
-      @root.insert(key, value)
-      @node_cache.flush
+      if @root.insert(key, value)
+        @size += 1
+        @node_cache.flush
+      end
     end
 
     # Retrieve the value associated with the given key. If no entry was found,
@@ -152,10 +233,17 @@ module PEROBS
       @root.get(key)
     end
 
+    # Either return the key/value pair that exactly matches the key or a
+    # key/value pair that has a key that is at least min_miss_increment larger
+    # than the key.
+    def get_best_match(key, min_miss_increment)
+      @root.get_best_match(key, min_miss_increment)
+    end
+
     # Find and remove the value associated with the given key. If no entry was
     # found, return nil, otherwise the found value.
     def remove(key)
-      removed_value = @root.remove(key)
+      @size -= 1 unless (removed_value = @root.remove(key)).nil?
 
       # Check if the root node only contains one child link after the delete
       # operation. Then we can delete that node and pull the tree one level
@@ -178,17 +266,6 @@ module PEROBS
       @root.each(&block)
     end
 
-    # Mark the given node as being modified. This will cause the dirty_flag
-    # lock to be taken and the @is_dirty flag to be set.
-    # @param node [BTreeNode] node to mark
-    def mark_node_as_modified(node)
-      unless @is_dirty
-        @dirty_flag.lock
-        @is_dirty = true
-      end
-      @node_cache.mark_as_modified(node)
-    end
-
     # Delete the node at the given address in the BTree file.
     # @param address [Integer] address in file
     def delete_node(address)
@@ -196,37 +273,16 @@ module PEROBS
       @nodes.delete_blob(address)
     end
 
+    # @return [Integer] The number of entries stored in the tree.
+    def entries_count
+      @size
+    end
+
     # @return [String] Human reable form of the tree.
     def to_s
       @root.to_s
     end
 
-    # Create a new BTreeNode. If the node_address is not nil, the node data is
-    # read from the backing store. The parent and is_leaf arguments are
-    # ignored in this case.
-    # @param parent [BTreeNode] parent node
-    # @param node_address [Integer or nil] address of the node to create
-    # @param is_leaf[Boolean] True if node is a leaf node, false otherweise
-    def new_node(parent, node_address = nil, is_leaf = true)
-      node = BTreeNode.new(self, parent, node_address, is_leaf)
-      @node_cache.insert(node)
-
-      node
-    end
-
-    # Return the BTreeNode that matches the given node address. If a blob
-    # address and size are given, a new node is created instead of being read
-    # from the file.
-    # @param node_address [Integer] Address of the node in the BTree file
-    # @return [BTreeNode]
-    def get_node(node_address)
-      if (node = @node_cache[node_address])
-        return node
-      end
-
-      new_node(nil, node_address)
-    end
-
   end
 
 end