perobs 2.5.0 → 3.0.0

data/lib/perobs/SpaceTreeNodeCache.rb

@@ -0,0 +1,76 @@
+# encoding: UTF-8
+#
+# = SpaceTree.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'perobs/SpaceTreeNode'
+
+module PEROBS
+
+  class SpaceTreeNodeCache
+
+    # Simple cache that can hold up to size SpaceTreeNode entries. Entries are
+    # hashed with a simple node_address % size function. This keeps the
+    # overhead for managing the cache extremely low yet giving an OK
+    # probability to have cache hits.
+    # @param size [Integer] maximum number of cache entries
+    def initialize(size)
+      @size = size
+      clear
+    end
+
+    # Insert a node into the cache.
+    # @param node [SpaceTreeNode]
+    def insert(node)
+      @entries[node.node_address % @size] = node
+    end
+
+    # Retrieve a node reference from the cache.
+    # @param address [Integer] address of the node to retrieve.
+    def get(address)
+      node = @entries[address % @size]
+      # We can have collisions. Check if the cached node really matches the
+      # requested address.
+      (node && node.node_address == address) ? node : nil
+    end
+
+    # Remove a node from the cache.
+    # @param address [Integer] address of node to remove.
+    def delete(address)
+      index = address % @size
+      if (node = @entries[index]) && node.node_address == address
+        @entries[index] = nil
+      end
+    end
+
+    # Remove all entries from the cache.
+    def clear
+      @entries = ::Array.new(@size)
+    end
+
+  end
+
+end
+

data/lib/perobs/SpaceTreeNodeLink.rb

@@ -0,0 +1,103 @@
+# encoding: UTF-8
+#
+# = SpaceTreeNodeLink.rb -- Persistent Ruby Object Store
+#
+# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+module PEROBS
+
+  # This class is used to form the links between the in-memory SpaceTreeNode
+  # objects. The link is based on the address of the node in the file. The
+  # class objects transparently convert the address into a corresponding
+  # SpaceTreeNode object and pass on all method calls.
+  class SpaceTreeNodeLink
+
+    attr_reader :node_address
+
+    # Create a new SpaceTreeNodeLink object.
+    # @param tree [SpaceTree] The SpaceTree that holds the nodes.
+    # @param node_or_address [SpaceTreeNode or SpaceTreeNodeLink or Integer] a
+    #        SpaceTreeNode, SpaceTreeNodeLink reference or the node
+    #        address in the file.
+    def initialize(tree, node_or_address)
+      @tree = tree
+      if node_or_address.is_a?(SpaceTreeNode) ||
+         node_or_address.is_a?(SpaceTreeNodeLink)
+        @node_address = node_or_address.node_address
+      elsif node_or_address.is_a?(Integer)
+        @node_address = node_or_address
+      else
+        PEROBS.log.fatal "Unsupported argument type #{node_or_address.class}"
+      end
+    end
+
+    # All calls to a SpaceTreeNodeLink object will be forwarded to the
+    # corresponding SpaceTreeNode object. If that
+    def method_missing(method, *args, &block)
+      get_node.send(method, *args, &block)
+    end
+
+    # Make it properly introspectable.
+    def respond_to?(method, include_private = false)
+      get_node.respond_to?(method)
+    end
+
+    # Compare this node to another node.
+    # @return [Boolean] true if node address is identical, false otherwise
+    def ==(node)
+      @node_address == node.node_address
+    end
+
+    # Compare this node to another node.
+    # @return [Boolean] true if node address is not identical, false otherwise
+    def !=(node)
+      @node_address != node.node_address
+    end
+
+    # Check the link to a sub-node. This method silently ignores all errors if
+    # the sub-node does not exist.
+    # @return [Boolean] True if link is OK, false otherweise
+    def check_node_link(branch, stack)
+      begin
+        return get_node.check_node_link(branch, stack)
+      rescue
+        return false
+      end
+    end
+
+    # @return Textual version of the SpaceTreeNode
+    def to_s
+      get_node.to_s
+    end
+
+    private
+
+    def get_node
+      @tree.get_node(@node_address)
+    end
+
+  end
+
+end
+

data/lib/perobs/Store.rb

@@ -347,28 +347,42 @@ module PEROBS
       # All objects must have in-db version.
       sync
       # Run basic consistency checks first.
-      @db.check_db(repair)
+      errors = @db.check_db(repair)
 
       # We will use the mark to mark all objects that we have checked already.
       # Before we start, we need to clear all marks.
       @db.clear_marks
 
-      errors = 0
       objects = 0
       @root_objects.each do |name, id|
         objects += 1
         errors += check_object(id, repair)
       end
+
+      # Delete all broken root objects.
+      if repair
+        @root_objects.delete_if do |name, id|
+          unless (res = @db.check(id, repair))
+            PEROBS.log.error "Discarding broken root object '#{name}' " +
+              "with ID #{id}"
+            errors += 1
+          end
+          !res
+        end
+      end
+
       if errors > 0
-        PEROBS.log.warn "#{errors} errors found in #{objects} objects"
+        if repair
+          PEROBS.log.error "#{errors} errors found in #{objects} objects"
+        else
+          PEROBS.log.fatal "#{errors} errors found in #{objects} objects"
+        end
       else
         PEROBS.log.debug "No errors found"
       end
-      # Delete all broken root objects.
-      @root_objects.delete_if { |name, id| !@db.check(id, repair) }
 
       # Ensure that any fixes are written into the DB.
-      sync
+      sync if repair
 
       errors
     end
@@ -504,7 +518,7 @@ module PEROBS
         # Get the next PEROBS object to check
         ref_obj, id = todo_list.pop
 
-        if (obj = object_by_id(id)) && @db.check(id, repair)
+        if (obj = object_by_id(id)) && (obj_ok = @db.check(id, repair))
           # The object exists and is OK. Mark is as checked.
           @db.mark(id)
           # Now look at all other objects referenced by this object.
@@ -517,15 +531,15 @@ module PEROBS
           # Remove references to bad objects.
           if ref_obj
             if repair
-              PEROBS.log.warn "Eliminating broken reference to object #{id} " +
-                "in object #{ref_obj._id}:\n" + ref_obj.inspect
+              PEROBS.log.error "Removing reference to " +
+                "#{obj ? 'broken' : 'non-existing'} object #{id} from:\n" +
+                ref_obj.inspect
               ref_obj._delete_reference_to_id(id)
             else
-              PEROBS.log.fatal "The following object references a " +
-                "non-existing object #{id}:\n" + ref_obj.inspect
+              PEROBS.log.error "The following object references a " +
+                "#{obj ? 'broken' : 'non-existing'} object #{id}:\n" +
+                ref_obj.inspect
             end
-          else
-            PEROBS.log.warn "Eliminating root object #{id}"
           end
           errors += 1
         end

data/lib/perobs/version.rb

@@ -1,4 +1,4 @@
 module PEROBS
   # The version number
-  VERSION = "2.5.0"
+  VERSION = "3.0.0"
 end

data/test/BTree_spec.rb

@@ -0,0 +1,128 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'fileutils'
+
+require 'spec_helper'
+require 'perobs/BTree'
+
+describe PEROBS::BTree do
+
+  before(:all) do
+    @db_dir = generate_db_name('BTree')
+    FileUtils.mkdir_p(@db_dir)
+    @m = PEROBS::BTree.new(@db_dir, 'btree', 11)
+  end
+
+  after(:all) do
+    FileUtils.rm_rf(@db_dir)
+  end
+
+  it 'should gracefully handle calling close when not open' do
+    @m.close
+  end
+
+  it 'should open the BTree' do
+    @m.open
+    expect(@m.to_s).to eql("o--- @1\n")
+    #expect(@m.to_a).to eql([])
+  end
+
+  it 'should support adding sequential key/value pairs' do
+    0.upto(100) do |i|
+      @m.insert(i, 2 * i)
+      expect(@m.check).to be true
+      expect(@m.get(i)).to eql(2 * i)
+    end
+  end
+
+  it 'should persist the data' do
+    @m.close
+    @m.open
+    expect(@m.check).to be true
+    0.upto(100) do |i|
+      expect(@m.get(i)).to eql(2 * i)
+    end
+  end
+
+  it 'should iterate over the stored key and value pairs' do
+    i = 0
+    @m.each do |k, v|
+      expect(k).to eql(i)
+      expect(v).to eql(2 * i)
+      i += 1
+    end
+    expect(i).to eql(101)
+  end
+
+  it 'should yield the key/value pairs on check' do
+    i = 0
+    @m.check do |k, v|
+      expect(k).to eql(k)
+      expect(v).to eql(2 * k)
+      i += 1
+    end
+    expect(i).to eql(101)
+  end
+
+  it 'should support clearing the tree' do
+    @m.clear
+    expect(@m.check).to be true
+  end
+
+  it 'should support erasing the backing store' do
+    @m.close
+    @m.erase
+    @m.open
+    expect(@m.check).to be true
+  end
+
+  it 'should support adding random key/value pairs' do
+    (1..1000).to_a.shuffle.each do |i|
+      @m.insert(i, i * 100)
+      expect(@m.check).to be true
+    end
+    (1..1000).to_a.shuffle.each do |i|
+      expect(@m.get(i)).to eql(i * 100)
+    end
+  end
+
+  it 'should support removing keys' do
+    @m.clear
+    @m.insert(1, 1)
+    expect(@m.remove(1)).to eql(1)
+    expect(@m.check).to be true
+    expect(@m.to_s).to eql("o--- @1\n")
+
+    (1..100).to_a.shuffle.each do |i|
+      @m.insert(i, i * 100)
+    end
+    (1..100).to_a.shuffle.each do |i|
+      expect(@m.remove(i)).to eql(i * 100)
+      expect(@m.check).to be true
+    end
+  end
+
+end

data/test/EquiBlobsFile_spec.rb

@@ -0,0 +1,199 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2016, 2017 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'fileutils'
+
+require 'spec_helper'
+require 'perobs/EquiBlobsFile'
+
+describe PEROBS::EquiBlobsFile do
+
+  before(:all) do
+    @db_dir = generate_db_name('EquiBlobsFile')
+    FileUtils.mkdir_p(@db_dir)
+    @bf = PEROBS::EquiBlobsFile.new(@db_dir, 'EquiBlobsFile', 8)
+  end
+
+  after(:all) do
+    FileUtils.rm_rf(@db_dir)
+  end
+
+  it 'should create the file' do
+    @bf.open
+    expect(@bf.total_entries).to eql(0)
+    expect(@bf.total_spaces).to eql(0)
+    expect(@bf.check).to be true
+  end
+
+  it 'should return free addresses' do
+    expect(@bf.free_address).to eql(1)
+  end
+
+  it 'should store and retrieve a blob' do
+    @bf.store_blob(1,'00000000')
+    expect(@bf.total_entries).to eql(1)
+    expect(@bf.check).to be true
+    expect(@bf.retrieve_blob(1)).to eql('00000000')
+    expect(@bf.check).to be true
+  end
+
+  it 'should store and retrieve multiple blobs' do
+    @bf.store_blob(1,'XXXXXXXX')
+    expect(@bf.total_entries).to eql(1)
+    expect(@bf.check).to be true
+    @bf.store_blob(2,'11111111')
+    expect(@bf.check).to be true
+    @bf.store_blob(3,'22222222')
+    expect(@bf.check).to be true
+    @bf.store_blob(4,'33333333')
+    expect(@bf.total_entries).to eql(4)
+    expect(@bf.check).to be true
+    expect(@bf.retrieve_blob(1)).to eql('XXXXXXXX')
+    expect(@bf.retrieve_blob(2)).to eql('11111111')
+    expect(@bf.retrieve_blob(3)).to eql('22222222')
+    expect(@bf.retrieve_blob(4)).to eql('33333333')
+  end
+
+  it 'should raise error for a too large address' do
+    PEROBS.log.open(StringIO.new)
+    expect { @bf.retrieve_blob(5) }.to raise_error(PEROBS::FatalError)
+    PEROBS.log.open($stderr)
+  end
+
+  it 'should delete entries' do
+    @bf.delete_blob(3)
+    expect(@bf.total_entries).to eql(3)
+    expect(@bf.total_spaces).to eql(1)
+    expect(@bf.check).to be true
+    @bf.delete_blob(2)
+    expect(@bf.total_entries).to eql(2)
+    expect(@bf.total_spaces).to eql(2)
+    expect(@bf.check).to be true
+    @bf.delete_blob(1)
+    expect(@bf.total_entries).to eql(1)
+    expect(@bf.total_spaces).to eql(3)
+    expect(@bf.check).to be true
+  end
+
+  it 'should raise error when inserting into a non-reserved cell' do
+    PEROBS.log.open(StringIO.new)
+    expect { @bf.store_blob(1,'XXXXXXXX') }.to raise_error(PEROBS::FatalError)
+    PEROBS.log.open($stderr)
+    expect(@bf.total_entries).to eql(1)
+    expect(@bf.check).to be true
+  end
+
+  it 'shoud support inserting into deleted cells' do
+    expect(@bf.free_address).to eql(1)
+    @bf.store_blob(1, '44444444')
+    expect(@bf.check).to be true
+  end
+
+  it 'should persist all values over and close/open' do
+    @bf.close
+    @bf.open
+    expect(@bf.total_entries).to eql(2)
+    expect(@bf.total_spaces).to eql(2)
+    expect(@bf.check).to be true
+
+    expect(@bf.retrieve_blob(1)).to eql('44444444')
+    expect(@bf.retrieve_blob(4)).to eql('33333333')
+  end
+
+  it 'should support inserting into deleted cells (2)' do
+    expect(@bf.free_address).to eql(2)
+    @bf.store_blob(2,'55555555')
+    expect(@bf.check).to be true
+
+    expect(@bf.free_address).to eql(3)
+    @bf.store_blob(3,'66666666')
+    expect(@bf.total_entries).to eql(4)
+
+    expect(@bf.check).to be true
+  end
+
+  it 'should support clearing the file' do
+    @bf.clear
+    expect(@bf.total_entries).to eql(0)
+    expect(@bf.total_spaces).to eql(0)
+    expect(@bf.check).to be true
+    expect(@bf.free_address).to eql(1)
+    @bf.store_blob(1,'00000000')
+    expect(@bf.total_entries).to eql(1)
+    expect(@bf.check).to be true
+    expect(@bf.retrieve_blob(1)).to eql('00000000')
+    expect(@bf.check).to be true
+  end
+
+  it 'should support trimming the file' do
+    @bf.clear
+    1.upto(8) do |i|
+      adr = @bf.free_address
+      @bf.store_blob(adr, (0.ord + i).chr * 8)
+    end
+    expect(@bf.total_entries).to eql(8)
+    @bf.delete_blob(1)
+    @bf.delete_blob(2)
+    @bf.delete_blob(4)
+    @bf.delete_blob(5)
+    @bf.delete_blob(7)
+    expect(@bf.total_entries).to eql(3)
+    expect(@bf.total_spaces).to eql(5)
+    expect(@bf.check).to be true
+
+    @bf.delete_blob(8)
+    expect(@bf.total_entries).to eql(2)
+    expect(@bf.total_spaces).to eql(4)
+    expect(@bf.check).to be true
+
+    @bf.delete_blob(6)
+    expect(@bf.total_entries).to eql(1)
+    expect(@bf.total_spaces).to eql(2)
+    expect(@bf.check).to be true
+
+    @bf.delete_blob(3)
+    expect(@bf.total_entries).to eql(0)
+    expect(@bf.total_spaces).to eql(0)
+    expect(@bf.check).to be true
+  end
+
+  it 'should not allow erase when open' do
+    expect{ capture_io{ @bf.erase } }.to raise_error(PEROBS::FatalError)
+  end
+
+  it 'should support erasing the file' do
+    @bf.close
+    @bf.erase
+    @bf.open
+    expect(@bf.total_entries).to eql(0)
+    expect(@bf.total_spaces).to eql(0)
+    expect(@bf.check).to be true
+    @bf.store_blob(1,'XXXXXXXX')
+    expect(@bf.total_entries).to eql(1)
+    expect(@bf.check).to be true
+  end
+
+end
+

data/test/FlatFileDB_spec.rb

@@ -27,22 +27,30 @@ require 'fileutils'
 
 require 'spec_helper'
 require 'perobs/FlatFileDB'
+require 'perobs/Store'
+
+class FlatFileDB_O < PEROBS::Object
+
+  po_attr :a, :b, :c
+
+  def initialize(store)
+    super
+    attr_init(:a, 'foo')
+    attr_init(:b, 42)
+    attr_init(:c, false)
+  end
+
+end
 
 describe PEROBS::FlatFileDB do
 
-  before(:all) do
-    @db_dir = generate_db_name('FlatFileDB')
+  before(:each) do
+    @db_dir = generate_db_name(__FILE__)
     FileUtils.mkdir_p(@db_dir)
-    @db = PEROBS::FlatFileDB.new(@db_dir)
-    @db.open
+    @store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
   end
 
   after(:each) do
-    @db.check_db
-  end
-
-  after(:all) do
-    @db.close
     FileUtils.rm_rf(@db_dir)
   end
 
@@ -57,5 +65,51 @@ describe PEROBS::FlatFileDB do
     expect { db2.open }.to raise_error(PEROBS::FatalError)
   end
 
+  it 'should do a version upgrade' do
+    # Close the store
+    @store['o'] = @store.new(FlatFileDB_O)
+    @store.exit
+
+    # Manually downgrade the version file to version 1
+    version_file = File.join(@db_dir, 'version')
+    File.write(version_file, '1')
+
+    # Open the store again
+    store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
+    expect(File.read(version_file).to_i).to eql(PEROBS::FlatFileDB::VERSION)
+    expect(store['o'].b).to eql(42)
+  end
+
+  it 'should refuse a version downgrade' do
+    # Close the store
+    @store.exit
+
+    # Manually downgrade the version file to version 1
+    version_file = File.join(@db_dir, 'version')
+    File.write(version_file, '1000000')
+
+    # Open the store again
+    expect { PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB) }.to raise_error(PEROBS::FatalError)
+  end
+
+  it 'should recover from a lost index file' do
+    @store['o'] = @store.new(FlatFileDB_O)
+    @store.exit
+
+    File.delete(File.join(@db_dir, 'index.blobs'))
+    store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
+    expect(store['o'].b).to eql(42)
+  end
+
+  it 'should repair a damaged index file' do
+    @store['o'] = @store.new(FlatFileDB_O)
+    @store.exit
+
+    File.write(File.join(@db_dir, 'index.blobs'), '*' * 500)
+    store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
+    store.check(true)
+    expect(store['o'].b).to eql(42)
+  end
+
 end