perobs 4.1.0 → 4.2.0

data/lib/perobs/SpaceTree.rb

@@ -54,7 +54,7 @@ module PEROBS
 
       # Benchmark runs showed a cache size of 128 to be a good compromise
       # between read and write performance trade-offs and memory consumption.
-      @cache = PersistentObjectCache.new(128, 5000, SpaceTreeNode, self)
+      @cache = PersistentObjectCache.new(256, -1, SpaceTreeNode, self)
     end
 
     # Open the SpaceTree file.

data/lib/perobs/Store.rb

@@ -46,8 +46,9 @@ require 'perobs/ConsoleProgressMeter'
 # PErsistent Ruby OBject Store
 module PEROBS
 
-  Statistics = Struct.new(:in_memory_objects, :root_objects,
-                          :marked_objects, :swept_objects)
+  Statistics = Struct.new(:in_memory_objects, :root_objects, :zombie_objects,
+                          :marked_objects, :swept_objects,
+                          :created_objects, :collected_objects)
 
   # PEROBS::Store is a persistent storage system for Ruby objects. Regular
   # Ruby objects are transparently stored in a back-end storage and retrieved
@@ -109,6 +110,7 @@ module PEROBS
   class Store
 
     attr_reader :db, :cache, :class_map
+    attr_writer :root_objects
 
     # Create a new Store.
     # @param data_base [String] the name of the database
@@ -143,6 +145,9 @@ module PEROBS
     #                      It defaults to ProgressMeter which only logs into
     #                      the log. Use ConsoleProgressMeter or a derived
     #                      class for more fancy progress reporting.
+    #   :no_root_objects : Create a new store without root objects. This only
+    #                      makes sense if you want to copy the objects of
+    #                      another store into this store.
     def initialize(data_base, options = {})
       # Create a backing store handler
       @progressmeter = (options[:progressmeter] ||= ProgressMeter.new)
@@ -155,25 +160,32 @@ module PEROBS
       # List of PEROBS objects that are currently available as Ruby objects
       # hashed by their ID.
       @in_memory_objects = {}
+      # List of objects that were destroyed already but were still found in
+      # the in_memory_objects list. _collect has not yet been called for them.
+      @zombie_objects = {}
 
       # This objects keeps some counters of interest.
       @stats = Statistics.new
+      @stats[:created_objects] = 0
+      @stats[:collected_objects] = 0
 
       # The Cache reduces read and write latencies by keeping a subset of the
       # objects in memory.
       @cache = Cache.new(options[:cache_bits] || 16)
 
       # The named (global) objects IDs hashed by their name
-      unless (@root_objects = object_by_id(0))
-        PEROBS.log.debug "Initializing the PEROBS store"
-        # The root object hash always has the object ID 0.
-        @root_objects = _construct_po(Hash, 0)
-        # Mark the root_objects object as modified.
-        @cache.cache_write(@root_objects)
-      end
-      unless @root_objects.is_a?(Hash)
-        PEROBS.log.fatal "Database corrupted: Root objects must be a Hash " +
-          "but is a #{@root_objects.class}"
+      unless options[:no_root_objects]
+        unless (@root_objects = object_by_id(0))
+          PEROBS.log.debug "Initializing the PEROBS store"
+          # The root object hash always has the object ID 0.
+          @root_objects = _construct_po(Hash, 0)
+          # Mark the root_objects object as modified.
+          @cache.cache_write(@root_objects)
+        end
+        unless @root_objects.is_a?(Hash)
+          PEROBS.log.fatal "Database corrupted: Root objects must be a Hash " +
+            "but is a #{@root_objects.class}"
+        end
       end
     end
 
@@ -185,7 +197,9 @@ module PEROBS
       sync
 
       # Create a new store with the specified directory and options.
-      new_db = Store.new(dir, options)
+      new_options = options.clone
+      new_options[:no_root_objects] = true
+      new_db = Store.new(dir, new_options)
       # Clear the cache.
       new_db.sync
       # Copy all objects of the existing store to the new store.
@@ -196,6 +210,7 @@ module PEROBS
         obj._sync
         i += 1
       end
+      new_db.root_objects = new_db.object_by_id(0)
       PEROBS.log.debug "Copied #{i} objects into new database at #{dir}"
       # Flush the new store and close it.
       new_db.exit
@@ -203,7 +218,6 @@ module PEROBS
       true
     end
 
-
     # Close the store and ensure that all in-memory objects are written out to
     # the storage backend. The Store object is no longer usable after this
     # method was called.
@@ -216,10 +230,22 @@ module PEROBS
       end
       @cache.flush if @cache
       @db.close if @db
-      @db = @class_map = @in_memory_objects = @stats = @cache = @root_objects =
-        nil
-    end
 
+      GC.start
+      if @stats
+        unless @stats[:created_objects] == @stats[:collected_objects] +
+            @in_memory_objects.length
+          PEROGS.log.fatal "Created objects count " +
+            "(#{@stats[:created_objects]})" +
+            " is not equal to the collected count " +
+            "(#{@stats[:collected_objects]}) + in_memory_objects count " +
+            "(#{@in_memory_objects.length})"
+        end
+      end
+
+      @db = @class_map = @in_memory_objects = @zombie_objects =
+        @stats = @cache = @root_objects = nil
+    end
 
     # You need to call this method to create new PEROBS objects that belong to
     # this Store.
@@ -255,8 +281,8 @@ module PEROBS
     # deletes the entire database.
     def delete_store
       @db.delete_database
-      @db = @class_map = @in_memory_objects = @stats = @cache = @root_objects =
-        nil
+      @db = @class_map = @in_memory_objects = @zombie_objects =
+        @stats = @cache = @root_objects = nil
     end
 
     # Store the provided object under the given name. Use this to make the
@@ -358,10 +384,10 @@ module PEROBS
         rescue RangeError => e
           # Due to a race condition the object can still be in the
           # @in_memory_objects list but has been collected already by the Ruby
-          # GC. In that case we need to load it again. In this case the
-          # _collect() call will happen much later, potentially after we have
-          # registered a new object with the same ID.
-          @in_memory_objects.delete(id)
+          # GC. In that case we need to load it again. The _collect() call
+          # will happen much later, potentially after we have registered a new
+          # object with the same ID.
+          @zombie_objects[id] = @in_memory_objects.delete(id)
         end
       end
 
@@ -510,7 +536,16 @@ module PEROBS
     # @param obj [BasicObject] Object to register
     # @param id [Integer] object ID
     def _register_in_memory(obj, id)
+      unless obj.is_a?(ObjectBase)
+        PEROBS.log.fatal "You can only register ObjectBase objects"
+      end
+      if @in_memory_objects.include?(id)
+        PEROBS.log.fatal "The Store::_in_memory_objects list already " +
+          "contains an object for ID #{id}"
+      end
+
       @in_memory_objects[id] = obj.object_id
+      @stats[:created_objects] += 1
     end
 
     # Remove the object from the in-memory list. This is an internal method
@@ -520,6 +555,10 @@ module PEROBS
     def _collect(id, ruby_object_id)
       if @in_memory_objects[id] == ruby_object_id
         @in_memory_objects.delete(id)
+        @stats[:collected_objects] += 1
+      elsif @zombie_objects[id] == ruby_object_id
+        @zombie_objects.delete(id)
+        @stats[:collected_objects] += 1
       end
     end
 
@@ -527,6 +566,7 @@ module PEROBS
     def statistics
       @stats.in_memory_objects = @in_memory_objects.length
       @stats.root_objects = @root_objects.length
+      @stats.zombie_objects = @zombie_objects.length
 
       @stats
     end
@@ -556,8 +596,10 @@ module PEROBS
     # Sweep phase of a mark-and-sweep garbage collector. It will remove all
     # unmarked objects from the store.
     def sweep
-      @stats.swept_objects = @db.delete_unmarked_objects
-      @cache.reset
+      @stats.swept_objects = @db.delete_unmarked_objects do |id|
+        @cache.evict(id)
+      end
+      GC.start
       PEROBS.log.debug "#{@stats.swept_objects} objects collected"
       @stats.swept_objects
     end

data/lib/perobs/version.rb

@@ -1,4 +1,4 @@
 module PEROBS
   # The version number
-  VERSION = "4.1.0"
+  VERSION = "4.2.0"
 end

data/perobs.gemspec

@@ -16,9 +16,9 @@ GEM_SPEC = Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = '>=2.3'
+  spec.required_ruby_version = '>=2.4'
 
   spec.add_development_dependency 'bundler', '~> 2.3'
   spec.add_development_dependency 'yard', '~>0.9.12'
-  spec.add_development_dependency 'rake', '~> 10.1'
+  spec.add_development_dependency 'rake', '~> 12.3.3'
 end

data/test/BTree_spec.rb

@@ -51,6 +51,7 @@ describe PEROBS::BTree do
     @m.open
     expect(@m.to_s).to eql("o--- @1\n")
     #expect(@m.to_a).to eql([])
+    expect(@m.check).to be true
   end
 
   it 'should support adding sequential key/value pairs' do

data/test/BigArray_spec.rb

@@ -32,7 +32,7 @@ require 'spec_helper'
 require 'perobs'
 require 'perobs/BigArray'
 
-NODE_ENTRIES = 4
+NODE_ENTRIES = 6
 
 describe PEROBS::BigArray do
 
@@ -59,12 +59,16 @@ describe PEROBS::BigArray do
     expect(@a.length).to eq(0)
     expect(@a.check).to be true
     expect(@a[0]).to be nil
+    expect(@a.first).to be nil
+    expect(@a.last).to be nil
   end
 
   it 'should append the first element' do
     @a << 0
     expect(@a.empty?).to be false
     expect(@a[0]).to eq(0)
+    expect(@a.first).to eq(0)
+    expect(@a.last).to eq(0)
     expect(@a.length).to eq(1)
     expect(@a.check).to be true
   end
@@ -75,6 +79,8 @@ describe PEROBS::BigArray do
       expect(@a.check).to be true
       expect(@a.length).to eq(i + 1)
     end
+    expect(@a.first).to eq(0)
+    expect(@a.last).to eq(10 * NODE_ENTRIES - 1)
   end
 
   it 'should insert at 0' do
@@ -111,7 +117,7 @@ describe PEROBS::BigArray do
 
   it 'should support the [] operator' do
     expect(@a[0]).to be nil
-    expect{@a[-1]}.to raise_error(IndexError)
+    expect(@a[-1]).to be nil
     @a[0] = 0
     expect(@a[0]).to eq(0)
     1.upto(3 * NODE_ENTRIES) do |i|
@@ -124,7 +130,7 @@ describe PEROBS::BigArray do
     0.upto(3 * NODE_ENTRIES) do |i|
       expect(@a[-3 * NODE_ENTRIES + i - 1]).to eq(i)
     end
-    expect{@a[-3 * NODE_ENTRIES - 2]}.to raise_error(IndexError)
+    expect(@a[-3 * NODE_ENTRIES - 2]).to be nil
     (3 * NODE_ENTRIES + 1).upto(4 * NODE_ENTRIES) do |i|
       expect(@a[i]).to be nil
     end
@@ -142,7 +148,7 @@ describe PEROBS::BigArray do
     expect(@a.length).to eq(0)
     expect(@a.check).to be true
 
-    n = 3 * NODE_ENTRIES
+    n = 5 * NODE_ENTRIES
     0.upto(n) { |i| @a.insert(i, i) }
     0.upto(n) do |i|
       expect(@a.delete_at(0)).to eql(i)
@@ -155,20 +161,52 @@ describe PEROBS::BigArray do
       expect(@a.check).to be true
     end
 
-    n = 11 * NODE_ENTRIES
+    n = 15 * NODE_ENTRIES
     0.upto(n - 1) { |i| @a.insert(i, i) }
     expect(@a.delete_at(n + 2)).to be nil
     expect(@a.delete_at(-(n + 2))).to be nil
     expect(@a.size).to eql(n)
 
     n.times do |i|
-      @a.delete_at(rand(@a.size))
+      idx = rand(@a.size)
+      @a.delete_at(idx)
       expect(@a.size).to be (n - 1 - i)
       expect(@a.check).to be true
     end
     expect(@a.size).to eql(0)
   end
 
+  it 'should fill the gaps' do
+    1.upto(4) do |i|
+      idx = i * NODE_ENTRIES * NODE_ENTRIES
+      @a[idx] = idx
+      expect(@a[idx - 1]).to be nil
+      expect(@a[idx + 1]).to be nil
+      expect(@a.check).to be true
+    end
+    expect(@a[0]).to be nil
+  end
+
+  it 'should insert after a gap' do
+    ref = []
+    10.times do |i|
+      idx = 10 + i * 3
+      @a[idx] = idx
+      ref[idx] = idx
+      expect(@a[idx]).to eql(idx)
+      expect(@a.check).to be true
+    end
+    10.times do |i|
+      idx = i * 5
+      @a[idx] = idx
+      ref[idx] = idx
+      expect(@a[idx]).to eql(idx)
+      expect(@a.check).to be true
+    end
+    expect(@a.check).to be true
+    expect(@a.to_a).to eql(ref)
+  end
+
   it 'should iterate over all values' do
     n = 3 * NODE_ENTRIES
     0.upto(n) { |i| @a.insert(i, i) }
@@ -191,6 +229,15 @@ describe PEROBS::BigArray do
     end
   end
 
+  it 'should insert at the beginning' do
+    (5 * NODE_ENTRIES).downto(0) do |i|
+      @a.insert(0, i)
+    end
+    expect(@a.check).to be true
+    a = Array.new(5 * NODE_ENTRIES + 1) { |i| i }
+    expect(@a.to_a).to eq(a)
+  end
+
   it 'should persist the data' do
     db_name = generate_db_name(__FILE__ + "_persist")
     store = PEROBS::Store.new(db_name)

data/test/BigHash_spec.rb

@@ -72,6 +72,13 @@ describe PEROBS::Hash do
     expect(@h.keys).to eql([ 'foo' ])
   end
 
+  it 'should store a few objects' do
+    20.times do |i|
+      @h["bar#{i}"] = "foo#{i}"
+    end
+    expect(@h.size).to eql(20)
+  end
+
   it 'should return nil for unknown objects' do
     expect(@h['bar']).to be_nil
   end
@@ -127,6 +134,7 @@ describe PEROBS::Hash do
     end
     expect(h.check).to be true
     expect(h.length).to eql(n)
+    expect(store.check).to eql(0)
     store.exit
 
     store = PEROBS::Store.new(db_name)

data/test/FlatFileDB_spec.rb

@@ -71,7 +71,7 @@ describe PEROBS::FlatFileDB do
     expect { db2.open }.to raise_error(PEROBS::FatalError)
   end
 
-  it 'should do a version upgrade' do
+  it 'should do a version upgrade from version 3' do
     # Close the store
     @store.exit
     src_dir = File.join(File.dirname(__FILE__), 'LegacyDBs', 'version_3')
@@ -82,6 +82,21 @@ describe PEROBS::FlatFileDB do
     capture_io { expect(db.check).to be true }
   end
 
+  it 'should do a version upgrade from version 4.1' do
+    # Close the store
+    @store.exit
+    src_dir = File.join(File.dirname(__FILE__), 'LegacyDBs', 'version_4.1')
+    FileUtils.cp_r(Dir.glob(src_dir + '/*'), @db_dir)
+
+    db = LegacyDB.new(@db_dir)
+    capture_io { db.open }
+    capture_io { expect(db.repair).to eql(0) }
+    expect(File.exist?(File.join(@db_dir, 'space_index.blobs'))).to be true
+    expect(File.exist?(File.join(@db_dir, 'space_list.blobs'))).to be true
+    expect(File.exist?(File.join(@db_dir, 'database_spaces.blobs'))).to be false
+    capture_io { expect(db.check).to be true }
+  end
+
   it 'should refuse a version downgrade' do
     # Close the store
     @store.exit
@@ -119,7 +134,7 @@ describe PEROBS::FlatFileDB do
     expect(store['o'].b).to eql(42)
   end
 
-  it 'should repair a corrupted database.blobs file' do
+  it 'should discard a corrupted blob inside the database.blobs file' do
     @store.exit
 
     db = PEROBS::FlatFileDB.new(@db_dir)
@@ -142,7 +157,7 @@ describe PEROBS::FlatFileDB do
     f.close
 
     db.open
-    expect(db.check_db).to eql(2)
+    expect(db.check_db).to eql(1)
     expect(db.check_db(true)).to eql(1)
     db.close
     db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
@@ -160,5 +175,95 @@ describe PEROBS::FlatFileDB do
     db.close
   end
 
+  it 'should discard a corrupted blob at the end of the database.blobs file' do
+    @store.exit
+
+    db = PEROBS::FlatFileDB.new(@db_dir)
+    db_file = File.join(@db_dir, 'database.blobs')
+    db.open
+    0.upto(5) do |i|
+      db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
+    end
+    db.close
+
+    f = File.truncate(db_file, File.size(db_file) - 20)
+
+    db.open
+    expect(db.check_db).to eql(2)
+    expect(db.check_db(true)).to eql(2)
+    db.close
+    db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
+                                           :log_level => Logger::ERROR })
+    db.open
+    expect(db.check_db).to eql(0)
+
+    0.upto(4) do |i|
+      expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
+    end
+    expect(db.get_object(6)).to be_nil
+    db.close
+  end
+
+  it 'should discard a corrupted header at the end of the database.blobs file' do
+    @store.exit
+
+    db = PEROBS::FlatFileDB.new(@db_dir)
+    db_file = File.join(@db_dir, 'database.blobs')
+    db.open
+    0.upto(5) do |i|
+      db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
+    end
+    db.close
+
+    f = File.truncate(db_file, File.size(db_file) - 200)
+
+    db.open
+    expect(db.check_db).to eql(1)
+    expect(db.check_db(true)).to eql(1)
+    db.close
+    db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
+                                           :log_level => Logger::ERROR })
+    db.open
+    expect(db.check_db).to eql(0)
+
+    0.upto(4) do |i|
+      expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
+    end
+    expect(db.get_object(6)).to be_nil
+    db.close
+  end
+
+  it 'should handle a lost blob at the end of the database.blobs file' do
+    @store.exit
+
+    db = PEROBS::FlatFileDB.new(@db_dir)
+    db_file = File.join(@db_dir, 'database.blobs')
+    db.open
+    0.upto(5) do |i|
+      db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
+    end
+    db.close
+
+    # This exactly removes the last blob (#6)
+    f = File.truncate(db_file, File.size(db_file) - 210)
+
+    db.open
+    expect(db.check_db).to eql(1)
+    # The repair won't find the missing blob since the blob file is without
+    # errors.
+    expect(db.check_db(true)).to eql(0)
+    db.close
+    db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
+                                           :log_level => Logger::ERROR })
+    db.open
+    expect(db.check_db).to eql(0)
+
+    0.upto(4) do |i|
+      expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
+    end
+    expect(db.get_object(6)).to be_nil
+    db.close
+  end
+
 end