perobs 3.0.1 → 4.3.0

data/test/EquiBlobsFile_spec.rb

@@ -27,13 +27,17 @@ require 'fileutils'
 
 require 'spec_helper'
 require 'perobs/EquiBlobsFile'
+require 'perobs/ProgressMeter'
 
 describe PEROBS::EquiBlobsFile do
 
   before(:all) do
+    PEROBS.log.level = Logger::ERROR
+    PEROBS.log.open($stderr)
     @db_dir = generate_db_name('EquiBlobsFile')
     FileUtils.mkdir_p(@db_dir)
-    @bf = PEROBS::EquiBlobsFile.new(@db_dir, 'EquiBlobsFile', 8)
+    @bf = PEROBS::EquiBlobsFile.new(@db_dir, 'EquiBlobsFile',
+                                    PEROBS::ProgressMeter.new, 8)
   end
 
   after(:all) do
@@ -134,6 +138,12 @@ describe PEROBS::EquiBlobsFile do
     expect(@bf.check).to be true
   end
 
+  it 'should support deleting reserved blobs' do
+    adr = @bf.free_address
+    @bf.delete_blob(adr)
+    expect(@bf.check).to be true
+  end
+
   it 'should support clearing the file' do
     @bf.clear
     expect(@bf.total_entries).to eql(0)
@@ -179,10 +189,6 @@ describe PEROBS::EquiBlobsFile do
     expect(@bf.check).to be true
   end
 
-  it 'should not allow erase when open' do
-    expect{ capture_io{ @bf.erase } }.to raise_error(PEROBS::FatalError)
-  end
-
   it 'should support erasing the file' do
     @bf.close
     @bf.erase
@@ -195,5 +201,99 @@ describe PEROBS::EquiBlobsFile do
     expect(@bf.check).to be true
   end
 
+  it 'should support custom offsets' do
+    @bf.close
+    @bf.erase
+    @bf.clear_custom_data
+    @bf.register_custom_data('foo', 42)
+    @bf.register_custom_data('bar', 43)
+    @bf.open
+    expect(@bf.total_entries).to eql(0)
+    expect(@bf.total_spaces).to eql(0)
+    expect(@bf.check).to be true
+    expect(@bf.free_address).to eql(1)
+    @bf.store_blob(1,'11111111')
+    expect(@bf.free_address).to eql(2)
+    @bf.store_blob(2,'22222222')
+    expect(@bf.free_address).to eql(3)
+    @bf.store_blob(3,'33333333')
+    expect(@bf.check).to be true
+    expect(@bf.total_entries).to eql(3)
+    expect(@bf.total_spaces).to eql(0)
+    @bf.delete_blob(2)
+    expect(@bf.check).to be true
+    expect(@bf.total_entries).to eql(2)
+    expect(@bf.total_spaces).to eql(1)
+    expect(@bf.free_address).to eql(2)
+    @bf.store_blob(2,'44444444')
+    expect(@bf.total_entries).to eql(3)
+    expect(@bf.total_spaces).to eql(0)
+    expect(@bf.check).to be true
+  end
+
+  it 'should support a mix of adds and deletes' do
+    @bf.close
+    @bf.erase
+    @bf.clear_custom_data
+    @bf.register_custom_data('foo', 42)
+    @bf.register_custom_data('bar', 43)
+    @bf.open
+
+    entries = {}
+    1000.times do |i|
+      rand(30).times do
+        adr = @bf.free_address
+        expect(entries[adr]).to be nil
+        val = rand(2 ** 64)
+        @bf.store_blob(adr, [ val ].pack('Q'))
+        entries[adr] = val
+        #expect(@bf.check).to be true
+      end
+
+      rand(15).times do
+        unless entries.empty?
+          addresses = entries.keys
+          adr = addresses[rand(addresses.length)]
+          expect(@bf.retrieve_blob(adr).unpack('Q').first).to eql(entries[adr])
+          @bf.delete_blob(adr)
+          entries.delete(adr)
+          #expect(@bf.check).to be true
+        end
+      end
+
+      rand(5).times do
+        unless entries.empty?
+          addresses = entries.keys
+          adr = addresses[rand(addresses.length)]
+          val = rand(2 ** 64)
+          @bf.store_blob(adr, [ val ].pack('Q'))
+          entries[adr] = val
+          #expect(@bf.check).to be true
+        end
+      end
+
+      if rand(100) == 0
+        expect(@bf.check).to be true
+      end
+
+      if rand(100) == 0
+        @bf.first_entry = i
+        @bf.set_custom_data('foo', i)
+      end
+
+      if rand(50) == 0
+        @bf.close
+        @bf.open
+      end
+
+      if rand(500) == 0
+        @bf.clear
+        entries = {}
+      end
+    end
+
+    expect(@bf.check).to be true
+  end
+
 end
 

data/test/FNV_Hash_1a_64_spec.rb

@@ -0,0 +1,59 @@
+# encoding: UTF-8
+#
+# Copyright (c) 2019 by Chris Schlaeger <chris@taskjuggler.org>
+#
+# This file contains tests for Array that are similar to the tests for the
+# Array implementation in MRI. The ideas of these tests were replicated in
+# this code.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require 'spec_helper'
+
+require 'perobs'
+
+describe PEROBS::FNV_Hash_1a_64 do
+
+  it 'should generate stable hashes for Strings' do
+    refs = [
+      [ 'foo', 15902901984413996407 ],
+      [ 'foo', 15902901984413996407 ],
+      [ 'bar', 16101355973854746 ],
+      [ 'foobar', 9625390261332436968 ],
+      [ 'PEROBS rocks your application!', 4089220442501866848 ],
+      [ 'Permission is hereby granted, free of charge, to any person ' +
+        'obtaining a copy of this software and associated documentation ' +
+        'files (the "Software"), to deal in the Software without ' +
+        'restriction, including without limitation the rights to use, ' +
+        'copy, modify, merge, publish, distribute, sublicense, and/or ' +
+        'sell copies of the Software, and to permit persons to whom the ' +
+        'Software is furnished to do so, subject to the following conditions:',
+        17637146001033534275 ]
+    ]
+
+    refs.each do |v|
+      expect(PEROBS::FNV_Hash_1a_64::digest(v[0])).to eql(v[1])
+    end
+  end
+
+end
+

data/test/FlatFileDB_spec.rb

@@ -28,10 +28,11 @@ require 'fileutils'
 require 'spec_helper'
 require 'perobs/FlatFileDB'
 require 'perobs/Store'
+require 'LegacyDBs/LegacyDB'
 
 class FlatFileDB_O < PEROBS::Object
 
-  po_attr :a, :b, :c
+  attr_persist :a, :b, :c
 
   def initialize(store)
     super
@@ -47,7 +48,12 @@ describe PEROBS::FlatFileDB do
   before(:each) do
     @db_dir = generate_db_name(__FILE__)
     FileUtils.mkdir_p(@db_dir)
-    @store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
+    @store_options = {
+      :engine => PEROBS::FlatFileDB,
+      :log => $stderr,
+      :log_level => Logger::ERROR
+    }
+    @store = PEROBS::Store.new(@db_dir, @store_options)
   end
 
   after(:each) do
@@ -65,19 +71,30 @@ describe PEROBS::FlatFileDB do
     expect { db2.open }.to raise_error(PEROBS::FatalError)
   end
 
-  it 'should do a version upgrade' do
+  it 'should do a version upgrade from version 3' do
     # Close the store
-    @store['o'] = @store.new(FlatFileDB_O)
     @store.exit
+    src_dir = File.join(File.dirname(__FILE__), 'LegacyDBs', 'version_3')
+    FileUtils.cp_r(Dir.glob(src_dir + '/*'), @db_dir)
 
-    # Manually downgrade the version file to version 1
-    version_file = File.join(@db_dir, 'version')
-    File.write(version_file, '1')
+    db = LegacyDB.new(@db_dir)
+    capture_io { db.open }
+    capture_io { expect(db.check).to be true }
+  end
 
-    # Open the store again
-    store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
-    expect(File.read(version_file).to_i).to eql(PEROBS::FlatFileDB::VERSION)
-    expect(store['o'].b).to eql(42)
+  it 'should do a version upgrade from version 4.1' do
+    # Close the store
+    @store.exit
+    src_dir = File.join(File.dirname(__FILE__), 'LegacyDBs', 'version_4.1')
+    FileUtils.cp_r(Dir.glob(src_dir + '/*'), @db_dir)
+
+    db = LegacyDB.new(@db_dir)
+    capture_io { db.open }
+    capture_io { expect(db.repair).to eql(0) }
+    expect(File.exist?(File.join(@db_dir, 'space_index.blobs'))).to be true
+    expect(File.exist?(File.join(@db_dir, 'space_list.blobs'))).to be true
+    expect(File.exist?(File.join(@db_dir, 'database_spaces.blobs'))).to be false
+    capture_io { expect(db.check).to be true }
   end
 
   it 'should refuse a version downgrade' do
@@ -89,7 +106,7 @@ describe PEROBS::FlatFileDB do
     File.write(version_file, '1000000')
 
     # Open the store again
-    expect { PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB) }.to raise_error(PEROBS::FatalError)
+    expect { PEROBS::Store.new(@db_dir) }.to raise_error(PEROBS::FatalError)
   end
 
   it 'should recover from a lost index file' do
@@ -97,7 +114,10 @@ describe PEROBS::FlatFileDB do
     @store.exit
 
     File.delete(File.join(@db_dir, 'index.blobs'))
-    store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
+    store = nil
+    capture_io do
+      store = PEROBS::Store.new(@db_dir)
+    end
     expect(store['o'].b).to eql(42)
   end
 
@@ -106,10 +126,174 @@ describe PEROBS::FlatFileDB do
     @store.exit
 
     File.write(File.join(@db_dir, 'index.blobs'), '*' * 500)
-    store = PEROBS::Store.new(@db_dir, :engine => PEROBS::FlatFileDB)
-    store.check(true)
+    store = nil
+    capture_io do
+      store = PEROBS::Store.new(@db_dir)
+    end
+    capture_io { store.check(true) }
     expect(store['o'].b).to eql(42)
   end
 
+  it 'should discard a corrupted blob inside the database.blobs file' do
+    @store.exit
+
+    db = PEROBS::FlatFileDB.new(@db_dir)
+    db_file = File.join(@db_dir, 'database.blobs')
+    db.open
+    0.upto(5) do |i|
+      db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}", i + 1)
+    end
+    db.close
+    db.open
+    0.upto(5) do |i|
+      db.put_object("#{i + 10}:#{'Y' * (i + 1) * 25}", i + 10)
+    end
+    pos = db.instance_variable_get('@flat_file').find_obj_addr_by_id(10)
+    db.close
+
+    f = File.open(db_file, 'rb+')
+    f.seek(pos)
+    f.write('ZZZZZ')
+    f.close
+
+    db.open
+    expect(db.check_db).to eql(1)
+    expect(db.check_db(true)).to eql(1)
+    db.close
+    db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
+                                           :log_level => Logger::ERROR })
+    db.open
+    expect(db.check_db).to eql(0)
+
+    0.upto(5) do |i|
+      expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}")
+    end
+    expect(db.get_object(10)).to be_nil
+    1.upto(5) do |i|
+      expect(db.get_object(i + 10)).to eql("#{i + 10}:#{'Y' * (i + 1) * 25}")
+    end
+    db.close
+  end
+
+  it 'should discard a corrupted blob at the end of the database.blobs file' do
+    @store.exit
+
+    db = PEROBS::FlatFileDB.new(@db_dir)
+    db_file = File.join(@db_dir, 'database.blobs')
+    db.open
+    0.upto(5) do |i|
+      db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
+    end
+    db.close
+
+    f = File.truncate(db_file, File.size(db_file) - 20)
+
+    db.open
+    expect(db.check_db).to eql(2)
+    expect(db.check_db(true)).to eql(2)
+    db.close
+    db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
+                                           :log_level => Logger::ERROR })
+    db.open
+    expect(db.check_db).to eql(0)
+
+    0.upto(4) do |i|
+      expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
+    end
+    expect(db.get_object(6)).to be_nil
+    db.close
+  end
+
+  it 'should discard a corrupted header at the end of the database.blobs file' do
+    @store.exit
+
+    db = PEROBS::FlatFileDB.new(@db_dir)
+    db_file = File.join(@db_dir, 'database.blobs')
+    db.open
+    0.upto(5) do |i|
+      db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
+    end
+    db.close
+
+    f = File.truncate(db_file, File.size(db_file) - 200)
+
+    db.open
+    expect(db.check_db).to eql(1)
+    expect(db.check_db(true)).to eql(1)
+    db.close
+    db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
+                                           :log_level => Logger::ERROR })
+    db.open
+    expect(db.check_db).to eql(0)
+
+    0.upto(4) do |i|
+      expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
+    end
+    expect(db.get_object(6)).to be_nil
+    db.close
+  end
+
+  it 'should handle a lost blob at the end of the database.blobs file' do
+    @store.exit
+
+    db = PEROBS::FlatFileDB.new(@db_dir)
+    db_file = File.join(@db_dir, 'database.blobs')
+    db.open
+    0.upto(5) do |i|
+      db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
+    end
+    db.close
+
+    # This exactly removes the last blob (#6)
+    f = File.truncate(db_file, File.size(db_file) - 210)
+
+    db.open
+    expect(db.check_db).to eql(1)
+    # The repair won't find the missing blob since the blob file is without
+    # errors.
+    expect(db.check_db(true)).to eql(0)
+    db.close
+    db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
+                                           :log_level => Logger::ERROR })
+    db.open
+    expect(db.check_db).to eql(0)
+
+    0.upto(4) do |i|
+      expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
+    end
+    expect(db.get_object(6)).to be_nil
+    db.close
+  end
+
+  it 'should handle duplicate entries for the same ID in database.blobs file' do
+    @store.exit
+
+    db = PEROBS::FlatFileDB.new(@db_dir)
+    db_file = File.join(@db_dir, 'database.blobs')
+    db.open
+    0.upto(5) do |i|
+      db.put_object("#{i + 1}:#{'X' * (i + 1) * 30}$", i + 1)
+    end
+    db.close
+
+    # This appends the entry 2 again
+    blob2 = File.read(db_file, 319 - 199, 199)
+    File.write(db_file, blob2, File.size(db_file))
+
+    db.open
+    expect(db.check_db).to eql(2)
+    expect(db.check_db(true)).to eql(1)
+    db.close
+    db = PEROBS::FlatFileDB.new(@db_dir, { :log => $stderr,
+                                           :log_level => Logger::WARN })
+    db.open
+    expect(db.check_db).to eql(0)
+
+    0.upto(5) do |i|
+      expect(db.get_object(i + 1)).to eql("#{i + 1}:#{'X' * (i + 1) * 30}$")
+    end
+    db.close
+  end
+
 end